diff --git a/docs/model-research/tuning.md b/docs/model-research/tuning.md index f4b37e0..c1f41f9 100644 --- a/docs/model-research/tuning.md +++ b/docs/model-research/tuning.md @@ -245,7 +245,7 @@ engine for the model class, then tune. runs; ExLlamaV3 and TensorFold are faster on short code and hold a smaller window (section 7). - Speed on one short request is not agent readiness. tqLLM decoded about 2.5x - faster than llama.cpp on the same GGUF and failed 8 of 11 multi-session checks + faster than llama.cpp on the same GGUF and passed at most 4 of 11 multi-session checks (section 11); stock SGLang at 25 tok/s passed 11 of 11. - The engine table is in [Engines](engines.md). diff --git a/oneshots/orrery/grades.tsv b/oneshots/orrery/grades.tsv index 12f29a0..c52af90 100644 --- a/oneshots/orrery/grades.tsv +++ b/oneshots/orrery/grades.tsv @@ -32,3 +32,6 @@ t5b2-qwen3.8-27b-vllm qwen3.8-27b-vllm 32000 length 32000 364.2 99.4 0 0 0 0 0 0 t5b3-qwen3.8-27b-vllm qwen3.8-27b-vllm 32000 length 32000 361.5 99.5 0 0 0 0 0 0 no SVG: length stop at 32,000 tokens, all thinking (five identical replies) t5b4-qwen3.8-27b-vllm qwen3.8-27b-vllm 32000 length 32000 361.4 99.4 0 0 0 0 0 0 no SVG: length stop at 32,000 tokens, all thinking (five identical replies) t5b5-qwen3.8-27b-vllm qwen3.8-27b-vllm 32000 length 32000 361.3 99.3 0 0 0 0 0 0 no SVG: length stop at 32,000 tokens, all thinking (five identical replies) +mx22-flashnext-heretic2-t5-oneshot-orrery-r1 lcpp6-flashnext-heretic2 140000 stop 33547 1513.4 22.3 0 0 0 0 0 0 Final model-supplied standalone SVG is malformed (mismatched tag, line 77). The harness also greedily concatenated two SVG blocks/prose: that packaging error was separated by extracting the final original codeblock verbatim, which still fails XML parsing. Initial valid background-only fragment is not substituted or graded. +t5nm1-lcpp6-ornith-122b-q4kxl lcpp6-ornith-122b-q4kxl 258048 stop 29646 1940.1 15.3 1 1 0 1 0 3 Parses and draws, but planets are left at the central origin instead of translated to arm ends. Only gears visibly move; outer markers stay fixed across sealed frames. Nominal period parameters are not functioning planetary orbits. Toothed wheels turn without credible meshing; no visible planetary day/night. +t5nm1-lcpp-flint-40b-q3kxl lcpp-flint-40b-q3kxl 258048 stop 30402 299.0 101.8 1 2 0 0 1 4 Renders with detached teeth and shadow halos; motion present; orbital periods 80/38/20/40/160/300s incorrectly order inner planets; no credible meshing pairs; offset black shadows rotate incorrectly. Independent original-source/frame review. diff --git a/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-10000.webp b/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-10000.webp new file mode 100644 index 0000000..256aad9 Binary files /dev/null and b/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-10000.webp differ diff --git a/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-1500.webp b/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-1500.webp new file mode 100644 index 0000000..256aad9 Binary files /dev/null and b/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-1500.webp differ diff --git a/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-5000.webp b/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-5000.webp new file mode 100644 index 0000000..256aad9 Binary files /dev/null and b/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frames/frame-5000.webp differ diff --git a/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/meta.json b/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/meta.json new file mode 100644 index 0000000..6fdce8f --- /dev/null +++ b/oneshots/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/meta.json @@ -0,0 +1,45 @@ +{ + "case": "mx22-flashnext-heretic2-t5-oneshot-orrery-r1", + "finish_reason": "stop", + "first_token_s": 5.7, + "max_tokens": 140000, + "model": "lcpp6-flashnext-heretic2", + "reasoning_chars": 75955, + "reply_chars": 22823, + "started": "2026-10-23T04:28:15-0500", + "static": { + "svg_found": true, + "parses": false, + "parse_error": "not well-formed (invalid token): line 120, column 0", + "bytes": 21251, + "animateTransform": 0, + "animate": 16, + "animateMotion": 0, + "keyframes": 4, + "script": 1, + "prose_chars_outside_svg": 1596 + }, + "status": "ok", + "test": "5", + "timings": { + "cache_n": 0, + "prompt_n": 101, + "prompt_ms": 2666.171, + "prompt_per_token_ms": 26.397732673267324, + "prompt_per_second": 37.882041324431185, + "predicted_n": 33547, + "predicted_ms": 1507725.559, + "predicted_per_token_ms": 44.94501755798009, + "predicted_per_second": 22.249407260993447 + }, + "tok_s": 22.3, + "usage": { + "completion_tokens": 33547, + "prompt_tokens": 101, + "total_tokens": 33648, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 1513.4 +} diff --git a/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-10000.webp b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-10000.webp new file mode 100644 index 0000000..715d4d9 Binary files /dev/null and b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-10000.webp differ diff --git a/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-1500.webp b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-1500.webp new file mode 100644 index 0000000..05764fd Binary files /dev/null and b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-1500.webp differ diff --git a/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-5000.webp b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-5000.webp new file mode 100644 index 0000000..b20cf10 Binary files /dev/null and b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/frames/frame-5000.webp differ diff --git a/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/meta.json b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/meta.json new file mode 100644 index 0000000..87e1a9f --- /dev/null +++ b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/meta.json @@ -0,0 +1,44 @@ +{ + "case": "t5nm1-lcpp-flint-40b-q3kxl", + "finish_reason": "stop", + "first_token_s": 0.2, + "max_tokens": 258048, + "model": "lcpp-flint-40b-q3kxl", + "reasoning_chars": 97785, + "reply_chars": 14723, + "started": "2026-10-22T01:08:14-0500", + "static": { + "svg_found": true, + "parses": true, + "bytes": 14242, + "animateTransform": 13, + "animate": 2, + "animateMotion": 0, + "keyframes": 0, + "script": 0, + "prose_chars_outside_svg": 481 + }, + "status": "ok", + "test": "test5-orrery", + "timings": { + "cache_n": 0, + "prompt_n": 51, + "prompt_ms": 67.447, + "prompt_per_token_ms": 1.3224901960784314, + "prompt_per_second": 756.1492727623171, + "predicted_n": 30402, + "predicted_ms": 298794.899, + "predicted_per_token_ms": 9.828456267885924, + "predicted_per_second": 101.7453781900072 + }, + "tok_s": 101.8, + "usage": { + "completion_tokens": 30402, + "prompt_tokens": 51, + "total_tokens": 30453, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 299.0 +} diff --git a/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/orrery.svg b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/orrery.svg new file mode 100644 index 0000000..3d056bc --- /dev/null +++ b/oneshots/orrery/t5nm1-lcpp-flint-40b-q3kxl/orrery.svg @@ -0,0 +1,207 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-10000.webp b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-10000.webp new file mode 100644 index 0000000..0f097a1 Binary files /dev/null and b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-10000.webp differ diff --git a/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-1500.webp b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-1500.webp new file mode 100644 index 0000000..c856d2b Binary files /dev/null and b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-1500.webp differ diff --git a/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-5000.webp b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-5000.webp new file mode 100644 index 0000000..dcb7b70 Binary files /dev/null and b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frames/frame-5000.webp differ diff --git a/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/meta.json b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/meta.json new file mode 100644 index 0000000..73406c5 --- /dev/null +++ b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/meta.json @@ -0,0 +1,44 @@ +{ + "case": "t5nm1-lcpp6-ornith-122b-q4kxl", + "finish_reason": "stop", + "first_token_s": 1.7, + "max_tokens": 258048, + "model": "lcpp6-ornith-122b-q4kxl", + "reasoning_chars": 87398, + "reply_chars": 15219, + "started": "2026-10-20T22:01:40-0500", + "static": { + "svg_found": true, + "parses": true, + "bytes": 14076, + "animateTransform": 20, + "animate": 0, + "animateMotion": 0, + "keyframes": 0, + "script": 0, + "prose_chars_outside_svg": 1146 + }, + "status": "ok", + "test": "test5-orrery", + "timings": { + "cache_n": 0, + "prompt_n": 59, + "prompt_ms": 1462.479, + "prompt_per_token_ms": 24.78777966101695, + "prompt_per_second": 40.34245961822358, + "predicted_n": 29646, + "predicted_ms": 1938393.583, + "predicted_per_token_ms": 65.3868639905549, + "predicted_per_second": 15.293591693653475 + }, + "tok_s": 15.3, + "usage": { + "completion_tokens": 29646, + "prompt_tokens": 59, + "total_tokens": 29705, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 1940.1 +} diff --git a/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/orrery.svg b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/orrery.svg new file mode 100644 index 0000000..7c27cdc --- /dev/null +++ b/oneshots/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/orrery.svg @@ -0,0 +1,184 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BRASS ORRERY · gear ratios after real planetary periods + 1 Earth revolution = 12 s · inner gears drive each arm · terminator = day / night + + \ No newline at end of file diff --git a/oneshots/pagoda/grades.tsv b/oneshots/pagoda/grades.tsv index bdced03..7f87e46 100644 --- a/oneshots/pagoda/grades.tsv +++ b/oneshots/pagoda/grades.tsv @@ -48,3 +48,6 @@ t6unc1-lcpp6-q38-coder390 lcpp6-q38-coder390 76000 stop 13568 350.4 54.6 0 0 0 0 t6unc1-lcpp6-thinkingcap-27b-heretic lcpp6-thinkingcap-27b-heretic 128000 stop 29400 869.6 38.1 0 0 0 0 0 0 blank: Uncaught SyntaxError Malformed arrow function parameter list (line 171); frames show only the caption t6unc1-q38-27b-cyber q38-27b-cyber 128000 stop 14340 346.3 42.9 0 0 0 0 0 0 blank: Uncaught SyntaxError missing ) after argument list (line 143); the page's own error badge shows it t6unc1-qwen3.8-27b-uncensored-vllm qwen3.8-27b-uncensored-vllm 140000 stop 86783 989.8 91.9 2 2 2 1 2 9 best of the ten: five red storeys with slate eaves and gold corner blocks raised, gold ringed spire, stepped stone base, red torii, three cherry trees, lanterns, voxel clouds; the pond is in the code but out of frame; OrbitControls autoRotate, petals and shadows visible +mx22-flashnext-heretic2-t6-oneshot-pagoda-r1 lcpp6-flashnext-heretic2 140000 stop 29545 1359.9 21.8 0 0 0 0 0 0 Original animation throws ReferenceError: clock is not defined before scene rendering; all sealed frames contain only the clear-color background. Offline three.js dependency packaging succeeds. Blank-page rule scores every dimension zero; no diagnostic model-code fix was scored. +t6nm1-lcpp6-ornith-122b-q4kxl lcpp6-ornith-122b-q4kxl 258048 stop 6653 523.0 15.4 1 2 1 2 2 8 Draws consistently from boxes; five tiers/upturned eaves but full spire is cropped in every frame. Platform, cherry trees, pond and torii visible. Falling petals, shadows and frame-based camera orbit present. +t6nm1-lcpp-flint-40b-q3kxl lcpp-flint-40b-q3kxl 258048 stop 15958 145.2 110.1 1 2 1 2 2 8 Draws consistently from boxes; five tiers but complete spire cropped in all three frames; platform pond cherry tree and partly occluded torii visible; petals shadows and frame-based camera orbit present. Offline CDN rewrite is packaging only; no model fix. diff --git a/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-10000.webp b/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-10000.webp new file mode 100644 index 0000000..21c7817 Binary files /dev/null and b/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-10000.webp differ diff --git a/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-1500.webp b/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-1500.webp new file mode 100644 index 0000000..21c7817 Binary files /dev/null and b/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-1500.webp differ diff --git a/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-5000.webp b/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-5000.webp new file mode 100644 index 0000000..21c7817 Binary files /dev/null and b/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frames/frame-5000.webp differ diff --git a/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/meta.json b/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/meta.json new file mode 100644 index 0000000..2cb5d0f --- /dev/null +++ b/oneshots/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/meta.json @@ -0,0 +1,47 @@ +{ + "case": "mx22-flashnext-heretic2-t6-oneshot-pagoda-r1", + "finish_reason": "stop", + "first_token_s": 4.3, + "max_tokens": 140000, + "model": "lcpp6-flashnext-heretic2", + "reasoning_chars": 79788, + "reply_chars": 13878, + "started": "2026-10-23T04:53:29-0500", + "static": { + "html_found": true, + "bytes": 13867, + "complete": true, + "three_module": true, + "importmap": true, + "umd_three": false, + "external_urls": [ + "https://cdn.jsdelivr.net/npm/three@0.186.1/build/three.module.js" + ], + "boxes": 2, + "instanced": false, + "prose_chars_outside_html": 11 + }, + "status": "ok", + "test": "6", + "timings": { + "cache_n": 42, + "prompt_n": 154, + "prompt_ms": 3809.691, + "prompt_per_token_ms": 24.738253246753246, + "prompt_per_second": 40.42322592567219, + "predicted_n": 29545, + "predicted_ms": 1355596.0, + "predicted_per_token_ms": 45.8839696723531, + "predicted_per_second": 21.794103848049126 + }, + "tok_s": 21.8, + "usage": { + "completion_tokens": 29545, + "prompt_tokens": 196, + "total_tokens": 29741, + "prompt_tokens_details": { + "cached_tokens": 42 + } + }, + "wall_s": 1359.9 +} diff --git a/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-10000.webp b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-10000.webp new file mode 100644 index 0000000..5ecc65c Binary files /dev/null and b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-10000.webp differ diff --git a/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-1500.webp b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-1500.webp new file mode 100644 index 0000000..39de992 Binary files /dev/null and b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-1500.webp differ diff --git a/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-5000.webp b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-5000.webp new file mode 100644 index 0000000..69fcbd4 Binary files /dev/null and b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frames/frame-5000.webp differ diff --git a/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/meta.json b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/meta.json new file mode 100644 index 0000000..b7a27b0 --- /dev/null +++ b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/meta.json @@ -0,0 +1,47 @@ +{ + "case": "t6nm1-lcpp-flint-40b-q3kxl", + "finish_reason": "stop", + "first_token_s": 0.3, + "max_tokens": 258048, + "model": "lcpp-flint-40b-q3kxl", + "reasoning_chars": 39763, + "reply_chars": 7436, + "started": "2026-10-22T01:05:49-0500", + "static": { + "html_found": true, + "bytes": 7425, + "complete": true, + "three_module": true, + "importmap": true, + "umd_three": false, + "external_urls": [ + "https://cdn.jsdelivr.net/npm/three@0.186.1/build/three.module.js" + ], + "boxes": 4, + "instanced": false, + "prose_chars_outside_html": 11 + }, + "status": "ok", + "test": "test6-pagoda", + "timings": { + "cache_n": 0, + "prompt_n": 142, + "prompt_ms": 249.918, + "prompt_per_token_ms": 1.7599859154929578, + "prompt_per_second": 568.1863651277619, + "predicted_n": 15958, + "predicted_ms": 144936.383, + "predicted_per_token_ms": 9.082934323494392, + "predicted_per_second": 110.09657940753219 + }, + "tok_s": 110.1, + "usage": { + "completion_tokens": 15958, + "prompt_tokens": 142, + "total_tokens": 16100, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 145.2 +} diff --git a/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/page.html b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/page.html new file mode 100644 index 0000000..ffcc4ad --- /dev/null +++ b/oneshots/pagoda/t6nm1-lcpp-flint-40b-q3kxl/page.html @@ -0,0 +1,250 @@ + + + + + + Voxel Japanese Pagoda + + + + + + + + diff --git a/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-10000.webp b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-10000.webp new file mode 100644 index 0000000..95fb92c Binary files /dev/null and b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-10000.webp differ diff --git a/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-1500.webp b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-1500.webp new file mode 100644 index 0000000..3121e6f Binary files /dev/null and b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-1500.webp differ diff --git a/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-5000.webp b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-5000.webp new file mode 100644 index 0000000..c9259c6 Binary files /dev/null and b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frames/frame-5000.webp differ diff --git a/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/meta.json b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/meta.json new file mode 100644 index 0000000..26b6dd5 --- /dev/null +++ b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/meta.json @@ -0,0 +1,48 @@ +{ + "case": "t6nm1-lcpp6-ornith-122b-q4kxl", + "finish_reason": "stop", + "first_token_s": 90.2, + "max_tokens": 258048, + "model": "lcpp6-ornith-122b-q4kxl", + "reasoning_chars": 1729, + "reply_chars": 16404, + "started": "2026-10-20T21:52:57-0500", + "static": { + "html_found": true, + "bytes": 16393, + "complete": true, + "three_module": true, + "importmap": true, + "umd_three": false, + "external_urls": [ + "https://cdn.jsdelivr.net/npm/three@0.186.1/build/three.module.js", + "https://cdn.jsdelivr.net/npm/three@0.186.1/examples/jsm/" + ], + "boxes": 2, + "instanced": false, + "prose_chars_outside_html": 11 + }, + "status": "ok", + "test": "test6-pagoda", + "timings": { + "cache_n": 0, + "prompt_n": 154, + "prompt_ms": 2178.123, + "prompt_per_token_ms": 14.143655844155845, + "prompt_per_second": 70.70307783352915, + "predicted_n": 6653, + "predicted_ms": 432826.366, + "predicted_per_token_ms": 65.06710252555622, + "predicted_per_second": 15.368749509127639 + }, + "tok_s": 15.4, + "usage": { + "completion_tokens": 6653, + "prompt_tokens": 154, + "total_tokens": 6807, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 523.0 +} diff --git a/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/page.html b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/page.html new file mode 100644 index 0000000..91e25a5 --- /dev/null +++ b/oneshots/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/page.html @@ -0,0 +1,562 @@ + + + + + +Voxel Japanese Pagoda + + + + + + + diff --git a/site/data/README.md b/site/data/README.md index d902395..767aec4 100644 --- a/site/data/README.md +++ b/site/data/README.md @@ -79,7 +79,7 @@ Fields marked *opt* may be missing or `null`. Types are what the page expects; i | Field | Type | Meaning | | --- | --- | --- | | `schema` | str | `bakeoff-site/1`. | -| `generated` | str, opt | When the source dashboard data was generated, shown under the hero stats. | +| `generated` | str, opt | When the newest source data was taken: history.json's `generated` when set, else the dashboard's. Shown under the hero stats. | | `built` | str, opt | When this file was built, shown next to `generated` with the newest run and finding dates. | | `title`, `tagline` | str | Hero title and one-line pitch (from `curated.json`). | | `links` | object | `repo`, `cards`, `method`, `conclusions`, `log`: repository URLs for the footer. Extra keys are shown too. | diff --git a/site/data/bakeoff.json b/site/data/bakeoff.json index 5d8092f..517a093 100644 --- a/site/data/bakeoff.json +++ b/site/data/bakeoff.json @@ -1,7 +1,7 @@ { "schema": "bakeoff-site/1", - "generated": "2026-10-15 14:36 CDT", - "built": "2026-10-22 19:48 CDT", + "generated": "2026-10-25 11:07 CDT", + "built": "2026-10-25 11:25 CDT", "title": "The Bake-off", "tagline": "Which open-weight models can do real agent work on one consumer GPU, how fast, and with what settings.", "links": { @@ -101,17 +101,198 @@ } ], "findings": [ + { + "date": "2026-10-24", + "title": "GPU unlock raises long-prompt prefill by 43%", + "kind": "engine", + "summary": "On Titan, vLLM Qwen3.8-27B was measured with GPU unlock off, then on, then on again. A 140826-token prompt prefilled at 736 tok/s off. Unlock rounds were 1053 and 1056 tok/s on seed 41, and 1053 and 1055 tok/s on seed 42. The probe summary calls that a 43.3% and 43.2% gain. Short code decode moved from 126.4 to 127.9 and 127.5 tok/s. The same check on llama.cpp 0.7 with Tiel left short code decode at 149.1, 148.6 and 148.8 tok/s.", + "settings": null, + "metrics": [ + { + "label": "vLLM prefill, 140826 tokens, seed 41", + "before": 736, + "after": 1053, + "unit": "tok/s" + }, + { + "label": "vLLM prefill, seed 41, unlock round 2", + "before": 736, + "after": 1056, + "unit": "tok/s" + }, + { + "label": "vLLM prefill, seed 42, unlock round 2", + "before": 736, + "after": 1055, + "unit": "tok/s" + }, + { + "label": "vLLM decode, 400-token code prompt, round 1", + "before": 126.4, + "after": 127.9, + "unit": "tok/s" + }, + { + "label": "vLLM deep decode, seed 41", + "before": 85.3, + "after": 94.5, + "unit": "tok/s" + }, + { + "label": "Tiel decode, 400-token code prompt, round 1", + "before": 149.1, + "after": 148.6, + "unit": "tok/s" + }, + { + "label": "Tiel prefill, 141004 tokens, seed 41, round 1", + "before": 2201, + "after": 2397, + "unit": "tok/s" + } + ], + "quality": "The needle check passed in every unlock phase, on and off.", + "caveat": "One off-round and two on-rounds, one card. Short decode barely moved. Tiel long-prompt prefill rose less (2201 to 2397 tok/s on seed 41, 1549 to 1746 on seed 42) and its deep decode stayed 97.7 tok/s on seed 41. The llama.cpp version is from the same day's entry spec, not from a spec file inside the probe directory." + }, + { + "date": "2026-10-23", + "title": "Flash-Next Heretic 2 passes the coding tests and fails both one-shots", + "kind": "model", + "summary": "Test 1 scored 5 of 5 checks, 36 tests and 0 failures at 18.2 tok/s. Test 2 built, initialized, exposed 7 tools, called a tool, and passed 34 tests with 0 failures. Three test-4 tickets passed, with 4, 13 and 11 hidden checks. The orrery one-shot scored 0 of 10 because the SVG did not parse. The pagoda one-shot scored 0 of 10 because every sealed frame was only the clear color.", + "settings": null, + "metrics": [ + { + "label": "Test 1 checks", + "before": null, + "after": 5, + "unit": "of 5" + }, + { + "label": "Test 2 tests passed", + "before": null, + "after": 34, + "unit": "tests" + }, + { + "label": "Orrery score", + "before": null, + "after": 0, + "unit": "of 10" + }, + { + "label": "Pagoda score", + "before": null, + "after": 0, + "unit": "of 10" + } + ], + "quality": "Code tests passed. One-shots did not render a usable scene.", + "caveat": "One run each. The trust-variant test 2 exposed 4 tools and passed 28 tests; that is a different case, not a failure of the 7-tool run." + }, + { + "date": "2026-10-23", + "title": "Ornith 122B and Defiance pass tests 1 and 2", + "kind": "model", + "summary": "Ornith UD-Q4_K_XL scored 5 of 5 CLI checks and 48 tests at 13.4 tok/s, then a 7-tool server with 43 tests and 0 failures at 12.9 tok/s. Defiance IQ4_XS at 196608 context scored 5 of 5 checks and 82 tests at 32.5 tok/s, then 7 tools, 46 tests and 0 failures at 22.6 tok/s.", + "settings": null, + "metrics": [ + { + "label": "Ornith test 1 decode", + "before": null, + "after": 13.4, + "unit": "tok/s" + }, + { + "label": "Ornith test 2 tests passed", + "before": null, + "after": 43, + "unit": "tests" + }, + { + "label": "Defiance test 1 decode", + "before": null, + "after": 32.5, + "unit": "tok/s" + }, + { + "label": "Defiance test 2 tests passed", + "before": null, + "after": 46, + "unit": "tests" + } + ], + "quality": "Both protocol suites passed. No new one-shot grade for either model in this window.", + "caveat": "One run each. Ornith has no spec snapshot, so context and KV are unknown. Defiance one-shots finished later and are not graded." + }, + { + "date": "2026-10-23", + "title": "Orca uncensored passes test 4; its trust tests did not compile", + "kind": "model", + "summary": "Test 1 scored 5 of 5 checks and 41 tests. The plain test 2 built, initialized, exposed 9 tools and passed 42 tests. Three test-4 tickets passed (4, 13 and 11 hidden checks). The trust run also built, initialized and called tools (9 tools), but cargo test exited 101 while compiling. The parsed 0 and 0 counts are not a suite pass.", + "settings": null, + "metrics": [ + { + "label": "Test 1 tests passed", + "before": null, + "after": 41, + "unit": "tests" + }, + { + "label": "Plain test 2 tests passed", + "before": null, + "after": 42, + "unit": "tests" + }, + { + "label": "Trust-run cargo test exit", + "before": null, + "after": 101, + "unit": "exit code" + } + ], + "quality": "Coding tickets passed. Trust-run test quality is incomplete, not a pass.", + "caveat": "The plain test-2 grade is in the prerequisite regrade file, not the main grade table. Test 3 for this model is not in the canonical grade table." + }, + { + "date": "2026-10-23", + "title": "Heretic 2 judges a known-good server right and its own server wrong", + "kind": "test", + "summary": "On the known-good server, MCP and REST were both 14 of 14 and the judgement was right (rubric 12). On its own server, REST was 14 of 14 and MCP 13 of 14. The fixed server builds, initializes, exposes 7 tools and passes 34 tests, but the missing fact is temperature and the final judgement is wrong (rubric 9).", + "settings": null, + "metrics": [ + { + "label": "Known-good rubric", + "before": null, + "after": 12, + "unit": "of 12" + }, + { + "label": "Own-server rubric", + "before": null, + "after": 9, + "unit": "of 12" + }, + { + "label": "Own-server MCP facts", + "before": null, + "after": 13, + "unit": "of 14" + } + ], + "quality": "Fact match was high. The own-server trust decision was wrong.", + "caveat": "One run of each flavor. Minutes use the run duration without model-load time." + }, { "date": "2026-10-22", - "title": "An expert cache doubles Flash-Next on one 3090", + "title": "An expert cache makes Flash-Next 1.5x faster on one 3090", "kind": "tuning", "summary": "Qwen3.8 Flash-Next (125B, 6B active) does not fit in 24 GB, so llama.cpp keeps experts in system RAM. Keeping all of them there (-cmoe) and giving the GPU a cache of hot experts (--moe-cache-mib), with q8 KV to make room, beat the old split of 40 layers on CPU on every GGUF tried, at 262k context.", "settings": "-cmoe --moe-cache-mib 12800 -ctk q8_0 -ctv q8_0 -b 2048 -ub 1024 -t 16", "metrics": [ { - "label": "Decode, orca IQ4_XS", + "label": "Decode, orca IQ4_XS (streamed, live config)", "before": 13, - "after": 24, + "after": 19, "unit": "tok/s" }, { @@ -146,7 +327,7 @@ } ], "quality": "Against the old f16 KV config: mean KL 0.0011 and 98 % top-1 agreement over 100k tokens of real code; two loads of the same config differ by KL 0.0002. Needles found at 100k, 190k and 250k.", - "caveat": "Lazy tensor reads (llama.cpp's default) make the first pass over unseen text slow when the page cache is cold; -lzm off fixes it but costs about 28 GB of extra RAM. The 13 to 24 tok/s for orca is the best arm of the sweep. With the shipped config the streamed figure is about 19 tok/s (25 to 37 raw). The tqLLM control run measured the same GGUF at about 40 tok/s on short outputs; that measure is higher than the streamed one." + "caveat": "Lazy tensor reads (llama.cpp's default) make the first pass over unseen text slow when the page cache is cold; -lzm off fixes it but costs about 28 GB of extra RAM. Raw decode with short outputs is higher (25-37 tok/s on orca). The tqLLM control run measured the same GGUF at about 40 tok/s on short outputs; that measure is higher than the streamed one." }, { "date": "2026-10-22", @@ -216,7 +397,7 @@ "date": "2026-10-22", "title": "tqLLM serves Flash-Next fast but not for agents", "kind": "engine", - "summary": "tqLLM 1.2.3 loaded the Flash-Next IQ3_S GGUF and decoded faster than llama.cpp on short benchmarks, but long prompts and several sessions failed.", + "summary": "tqLLM 1.2.3 decodes Flash-Next about 2.5x faster than llama.cpp on the same GGUF, but one prefill out-of-memory leaves every later request failing until a restart.", "settings": null, "metrics": [ { @@ -244,8 +425,8 @@ "unit": "of 11" } ], - "quality": "Tool calls and a 30k needle passed at 32k; the 262k arm answered long prompts with HTTP 500.", - "caveat": "Short outputs with thinking off flatter the speed; the best 262k arm streamed 54 tok/s and passed 4 of 11 sessions. First load took 23 minutes cold." + "quality": "Tool calls and a 30k needle passed at 32k; at 262k with 8k prefill chunks it found a needle in 248k tokens. At 131072 context it failed on a 289-token prompt, and every later request failed until a restart.", + "caveat": "Short replies with thinking off; any prompt of 256+ tokens can trigger the failure. First load took 23 minutes cold." }, { "date": "2026-10-21", @@ -991,6 +1172,34 @@ "title": "Expert cache, Strata 0.2.2, Flint and tqLLM", "what": "Flash-Next sped up on llama.cpp and Strata; a new MoE and a new engine were tried.", "why": "Make the 125B model usable on one card." + }, + { + "date": "2026-10-23", + "era": "Models", + "title": "Ornith 122B and Defiance pass tests 1 and 2", + "what": "Ornith at 13.4 tok/s and Defiance at 32.5 tok/s both cleared the CLI checks and the Rust server tests.", + "why": "Do the new large and uncensored quants finish a real task?" + }, + { + "date": "2026-10-23", + "era": "Flash-Next", + "title": "Heretic 2 passes the code tests", + "what": "Five CLI checks, a 7-tool server with 34 passing tests, and three maintenance tickets. Both one-shots scored 0.", + "why": "An abliterated Flash-Next quant had not been graded on this matrix." + }, + { + "date": "2026-10-23", + "era": "Harness", + "title": "A trust run whose tests did not compile", + "what": "Orca uncensored passed test 1, plain test 2, and three test-4 tickets. Its trust run's tests failed to compile, so 0 and 0 is not a pass.", + "why": "A green protocol line can hide a broken test build." + }, + { + "date": "2026-10-24", + "era": "Engines", + "title": "GPU unlock speeds long prefill", + "what": "On Titan, vLLM Qwen3.8-27B long-prompt prefill rose from 736 to about 1050 tok/s. Short decode stayed near 127 tok/s.", + "why": "Does unlocking the GPU clock help a model that already fits on the card?" } ], "events": [], @@ -2482,6 +2691,38 @@ "label": "no", "status": "none" } + }, + "lcpp6-ornith-122b-q4kxl": { + "id": "lcpp6-ornith-122b-q4kxl", + "family": "Ornith 1.5", + "label": "Ornith 1.5 122B-A10B UD-Q4_K_XL · llama.cpp 0.7, experts partly in RAM", + "base": "ornith-ai/Ornith-1.5-122B", + "arch": "MoE 122B-A10B", + "engine": "llama.cpp 0.7", + "quant": "UD-Q4_K_XL", + "kind": "tune", + "uncensored": false, + "note": "MIT. Fine-tune of Qwen3.5-122B-A10B. Thinking on. An offload sweep on 2026-10-20 with 41 expert layers in RAM and 2048-token batches recorded 16.1 tok/s decode and 392.1 tok/s prefill. The agent runs above are slower; context and KV for those runs were not in a spec snapshot.", + "vision": { + "label": "not surveyed", + "status": null + } + }, + "lcpp6-q38-defiance-192k-author": { + "id": "lcpp6-q38-defiance-192k-author", + "family": "Qwen3.8 27B", + "label": "Defiance IQ4_XS · llama.cpp 0.7, 196608 context, q8 KV, author sampler", + "base": "Qwen/Qwen3.8-27B", + "arch": "dense 27B", + "engine": "llama.cpp 0.7", + "quant": "IQ4_XS", + "kind": "uncensored-tune", + "uncensored": true, + "note": "Uncensored tune. Author sampler, no draft weights. This run served 196608 tokens with q8 KV.", + "vision": { + "label": "not surveyed", + "status": null + } } }, "runs": [ @@ -7428,346 +7669,730 @@ "loc": 9497, "errors_per_kloc": 0.4 } - } - ], - "test3": [ - { - "case": "t3g-qwen3.8-27b-vllm", - "model": "qwen3.8-27b-vllm", - "flavor": "3G", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "One probe session, nine numbered calls with exact arguments and fields, errors reported verbatim; built REST ground truth first, then checked the probe against REST and the shim wire log (\"the probe is an LLM and could have hallucinated\"); trusted the known-good server after a full source audit. 15 min, 49.5 tok/s, 2 compactions. Read /t3/shim.jsonl and /opt/t3/shim.py (harness visible to the agent).", - "tps": 49.5, - "minutes": 15, - "finished": "2026-10-07 19:15" - }, - { - "case": "t3o-qwen3.8-27b-vllm", - "model": "qwen3.8-27b-vllm", - "flavor": "3O", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "Own server (5 tools) answered every fact it could correctly but had no services tool; built REST ground truth, found the gap, added ha_list_services, extended tests to 79, rebuilt; fixed server grades builds, 6 tools, 80 tests. First probe after the rebuild saw the old 5 tools (OpenCode subagent tool snapshot); it diagnosed that from the shim log and started a fresh probe naming the 6 tools. Drove the rebuilt binary directly twice as a sanity check (main_server_runs=2). 13 min, 51.2 tok/s.", - "tps": 51.2, - "minutes": 13, - "finished": "2026-10-07 19:47" }, { - "case": "t3g-strata-flash-next-p2", - "model": "strata-flash-next-p2", - "flavor": "3G", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "One probe session with eight numbered calls naming exact arguments and fields, \"quote the error text exactly and continue\"; built REST ground truth with jq first and diffed every fact. Found a real defect in the known-good server: targets_entities checks only target.entity_id, but all 177 targeted services in this HA publish target.entity, so every service reported false. Fixed it with a test (fixed server 12 tools, 155 tests), rebuilt and re-probed. Drove the binary directly four times (main_server_runs=4). 21 min, 20.0 tok/s on two-slot Strata.", - "tps": 20.0, - "minutes": 21, - "finished": "2026-10-07 20:29" + "case": "mx22-orcarouter-fn-uncensored-t1-cli-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 41, + "tests_fail": 0, + "subcmds": 10, + "tps": 19.9, + "minutes": 29, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-22 21:40", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 861, + "errors_per_kloc": 0.0 + } }, { - "case": "t3o-strata-flash-next-p2", - "model": "strata-flash-next-p2", - "flavor": "3O", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "Three probe sessions: an eight-question brief with method notes (paginate, cross-check counts, quote errors), a verbatim cross-check through resources and tools, and a re-check after the fix. Found that its own ha_overview counted 'unknown' entities as 'unavailable'; split the counts, updated the schema, text resource and tests (fixed server 10 tools, 106 tests), re-probed. 17 min, 18.0 tok/s.", - "tps": 18.1, - "minutes": 17, - "finished": "2026-10-07 20:45" + "case": "mx22-orcarouter-fn-uncensored-t2t-rust-mcp-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": null, + "tests_fail": null, + "subcmds": null, + "tps": 13.9, + "minutes": 166, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "partial", + "finished": "2026-10-23 00:28", + "struggle": { + "compiles": 17, + "failed_compiles": 14, + "rust_errors": 60, + "loc": 2340, + "errors_per_kloc": 25.6 + } }, { - "case": "t3g-orcarouter-lcpp6", + "case": "mx22-orcarouter-fn-uncensored-setupretry1-t4-bugfix-targets-r1", "model": "lcpp6-orcarouter-fn-uncensored", - "flavor": "3G", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "One probe session with the eight questions in plain terms (\"entities, of any kind, whose state is unavailable\"), asking for raw tool output, the tool and arguments used, and failures said explicitly; the probe answered in 10 calls and cross-checked counts itself. Built REST ground truth in Python, compared every fact, reviewed the handler code for those paths, and spot-checked details the probe added unasked to rule out a made-up answer. Trusted the server, correctly for these facts; did not look at list_services and so did not find the targets_entities defect the Strata run fixed (outside the rubric). Leanest run so far: 12 min, 6.1k output tokens, 8.4 tok/s.", - "tps": 8.4, + "test": 4, + "agent": "OpenCode", + "checks": 4, + "tests_pass": 156, + "tests_fail": 0, + "subcmds": null, + "tps": 17.2, "minutes": 12, - "finished": "2026-10-07 21:04" + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 00:41", + "struggle": { + "compiles": 5, + "failed_compiles": 2, + "rust_errors": 2, + "loc": 9483, + "errors_per_kloc": 0.3 + } }, { - "case": "t3o-orcarouter-lcpp6", + "case": "mx22-orcarouter-fn-uncensored-setupretry1-t4-config-timeouts-r1", "model": "lcpp6-orcarouter-fn-uncensored", - "flavor": "3O", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "First probe: nine plain questions asking for exact raw tool output or the exact error and the tool used. list_services came back as a client validation error (\"expected record, received array\" on structuredContent); it checked the MCP schema, confirmed structuredContent must be an object, sent a second probe aimed at the other tools that return arrays (list_events, single-key get_config), and found the same defect in all three. Wrapped them in objects, added a guard in mcp.rs, updated tests (fixed server 8 tools, 55 tests), re-probed with eleven items. REST ground truth for every fact. 46 min, 5.8 tok/s (40 MoE layers on CPU).", - "tps": 5.8, - "minutes": 46, - "finished": "2026-10-07 21:53" + "test": 4, + "agent": "OpenCode", + "checks": 13, + "tests_pass": 170, + "tests_fail": 0, + "subcmds": null, + "tps": 17.4, + "minutes": 28, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 01:10", + "struggle": { + "compiles": 9, + "failed_compiles": 5, + "rust_errors": 9, + "loc": 9888, + "errors_per_kloc": 0.9 + } }, { - "case": "t3g-thinkingcap-heretic", - "model": "lcpp6-thinkingcap-27b-heretic", - "flavor": "3G", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "Read the whole source, built REST ground truth (caught and fixed a bug in its own script), then two probes in parallel, each a short list of exact tool calls with exact arguments, the one field to report per call, \"ABSENT\" markers for missing attributes and verbatim errors. Every value matched; trusted the server after the source read. 9 min, 19.8 tok/s, the fastest test 3 run.", - "tps": 19.8, - "minutes": 9, - "finished": "2026-10-07 22:02" + "case": "mx22-orcarouter-fn-uncensored-setupretry1-t4-feature-filter-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 4, + "agent": "OpenCode", + "checks": 11, + "tests_pass": 159, + "tests_fail": 0, + "subcmds": null, + "tps": 14.9, + "minutes": 12, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 01:23", + "struggle": { + "compiles": 2, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9613, + "errors_per_kloc": 0.1 + } }, { - "case": "t3o-thinkingcap-heretic", - "model": "lcpp6-thinkingcap-27b-heretic", - "flavor": "3O", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "Baseline probe named its own tools and asked for the text and the structured value per call, errors verbatim. Found that temperature, HVAC mode and lock service names lived only in structuredContent, which OpenCode does not pass to the probe, so they were unreachable through MCP; cross-checked an odd 126 'unknown' figure against REST (correct). Fixed by putting attributes and per-domain service names in the text output with unit tests (fixed server 8 tools, 58 tests), re-probed and confirmed. 15 min, 24.7 tok/s.", - "tps": 24.7, - "minutes": 15, - "finished": "2026-10-07 22:18" + "case": "mx22-flashnext-heretic2-t1-cli-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 36, + "tests_fail": 0, + "subcmds": 11, + "tps": 18.2, + "minutes": 86, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 02:52", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 1436, + "errors_per_kloc": 0.7 + } }, { - "case": "t3o-qwen3.8-27b-uncensored-vllm-pp", - "model": "qwen3.8-27b-uncensored-vllm", - "flavor": "3O", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "It measured the /api/states payload (94 KB) and split the probe work into one metadata session and three parallel per-domain scans, each returning a fixed JSON shape with verbatim errors; its own list_entities has no state filter, so the probe counted \"unavailable\" rows itself. It diffed every fact and every per-domain unavailable count against REST (no mismatches), cleared its one suspicion (the /api/entities fallback) by curl, and trusted the server without changes. 11 min, 42.3 tok/s.", - "tps": 42.3, - "minutes": 11, - "finished": "2026-10-09 09:39" + "case": "mx22-flashnext-heretic2-t2-rust-mcp-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 34, + "tests_fail": 0, + "subcmds": null, + "tps": 17.1, + "minutes": 91, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 04:28", + "struggle": { + "compiles": 23, + "failed_compiles": 9, + "rust_errors": 33, + "loc": 1752, + "errors_per_kloc": 18.8 + } }, { - "case": "t3g-qwen3.8-27b-uncensored-vllm-pp", - "model": "qwen3.8-27b-uncensored-vllm", - "flavor": "3G", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 10, - "note": "One probe session with twelve numbered calls giving exact arguments and the fields to copy verbatim; it built REST ground truth and all 14 facts matched, then re-probed after its change. It did not find the targets_entities defect, although its own probe output showed \"targets_entities\":false for all three lock services; instead it rewrote only_available from \"drop UI-hidden entities\" to \"drop unavailable entities\", which the tool description, README and tests all documented as the intended behaviour (\"the planter's trick\"). 18 min, 47.4 tok/s; read the head of /t3/shim.jsonl and ran /opt/t3/shim.py --help (harness visible to the agent).", - "tps": 47.4, - "minutes": 18, - "finished": "2026-10-09 09:27" + "case": "mx22-flashnext-heretic2-t4-bugfix-targets-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 4, + "agent": "OpenCode", + "checks": 4, + "tests_pass": 156, + "tests_fail": 0, + "subcmds": null, + "tps": 13.9, + "minutes": 10, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 05:27", + "struggle": { + "compiles": 4, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9440, + "errors_per_kloc": 0.1 + } }, { - "case": "t3g-orcasaq2-cyber-27b", - "model": "orcasaq2-cyber-27b", - "flavor": "3G", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "It sent one probe session with the eight questions in plain terms, asking for the exact tool calls, the raw values and the exact error text; it built REST ground truth in Python, matched all 14 facts, and then read the source. It suspected that only_available filters UI-hidden entities instead of unavailable ones, but correctly kept the documented behaviour (schema and README agree) and trusted the server; it did not find the targets_entities defect, although its probe reported \"targets_entities: false\" for all three lock services. After three context compactions it had lost the prompt's exact facts.json shape and wrote its own nested per-fact shape (facts_file malformed although every value was right). 16 min, 42.7 tok/s. Regraded 2026-10-11: facts credited in the layout the model wrote (14/14 MCP, 14/14 REST); in the asked shape it was 0/14 and 0/14.", - "tps": 42.7, - "minutes": 16, - "finished": "2026-10-09 09:56" + "case": "mx22-flashnext-heretic2-t4-config-timeouts-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 4, + "agent": "OpenCode", + "checks": 13, + "tests_pass": 166, + "tests_fail": 0, + "subcmds": null, + "tps": 17.2, + "minutes": 32, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 06:00", + "struggle": { + "compiles": 3, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9748, + "errors_per_kloc": 0.1 + } }, { - "case": "t3o-orcasaq2-cyber-27b", - "model": "orcasaq2-cyber-27b", - "flavor": "3O", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "It built REST ground truth and read its own six-file server before sending one probe session: ten numbered calls with exact tool names, arguments and the field to report, errors to be quoted, and a summary table at the end. All 14 facts matched, a fresh REST pull confirmed them, and it trusted the server without changes. Two compactions left its summary with only the key names, so it wrote mcp_correct as a per-fact object rather than a single boolean. 8 min, 45.5 tok/s.", - "tps": 45.5, - "minutes": 8, - "finished": "2026-10-09 10:04" + "case": "mx22-flashnext-heretic2-t4-feature-filter-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 4, + "agent": "OpenCode", + "checks": 11, + "tests_pass": 159, + "tests_fail": 0, + "subcmds": null, + "tps": 12.0, + "minutes": 15, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 06:15", + "struggle": { + "compiles": 3, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9535, + "errors_per_kloc": 0.1 + } }, { - "case": "t3g-qwen3.8-27b", - "model": "qwen3.8-27b", - "flavor": "3G", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "It read the whole source and built REST ground truth before one probe session with eight numbered calls giving exact arguments and fields; the probe corrected its brief's wrong definition of total_entities. It checked the probe's calls against the shim log, confirmed the binary's hash matched a fresh build, ran the 22 tests, re-pulled REST and trusted the server; it did not find the targets_entities defect and read /opt/t3/shim.py and /t3/shim.jsonl (harness visible to the agent). 16 min, 21.8 tok/s.", - "tps": 21.8, - "minutes": 16, - "finished": "2026-10-09 10:22" + "case": "mx22-flashnext-heretic2-t2t-rust-mcp-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 28, + "tests_fail": 0, + "subcmds": null, + "tps": 19.8, + "minutes": 51, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 07:10", + "struggle": { + "compiles": 13, + "failed_compiles": 4, + "rust_errors": 10, + "loc": 1818, + "errors_per_kloc": 5.5 + } }, { - "case": "t3o-qwen3.8-27b", - "model": "qwen3.8-27b", - "flavor": "3O", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "Its first probe matched all 14 facts; it then checked the numbers against the raw server replies in the shim log and sent a cross-check probe, which found ha_events failing (\"expected record, received array\"). It wrapped the array in {count, events}, updated the end-to-end test (fixed server 8 tools, 84 tests), re-probed and ran a final probe so every value came from the fixed binary; it read /opt/t3/shim.py and /t3/shim.jsonl (harness visible to the agent). 15 min, 19.6 tok/s.", - "tps": 19.6, - "minutes": 15, - "finished": "2026-10-09 10:38" + "case": "mx22-orcarouter-fn-uncensored-t2setupretry1-t2-rust-mcp-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 42, + "tests_fail": 0, + "subcmds": null, + "tps": 14.1, + "minutes": 152, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 09:44", + "struggle": { + "compiles": 44, + "failed_compiles": 20, + "rust_errors": 53, + "loc": 3337, + "errors_per_kloc": 15.9 + } }, { - "case": "t3g-tiel-coder-35b-q6", - "model": "tiel-coder-35b-q6", - "flavor": "3G", - "mcp_facts": "14/14", - "api_facts": "14/14", - "judgement_right": true, - "rubric": 12, - "note": "It sent one probe session with ten numbered calls giving exact arguments and the field it needed from each, asking for isError, the verbatim JSON text and a closing summary block; it built REST ground truth first, noticed that the probe's pasted state=off block was internally inconsistent and settled that, and the probe's clipped 124-entity reply (the server caps text at 24,000 characters while structuredContent stays complete), by driving the binary directly over JSON-RPC (main_server_runs=6). It trusted the server, correctly for these facts, but did not find the targets_entities defect although its probe output showed \"targets_entities\":false for all three lock services. 23 min, 22.4 tok/s.", - "tps": 22.4, - "minutes": 23, - "finished": "2026-10-09 11:05" - }, + "case": "mx22-ornith-122b-q4kxl-t1-cli-r1", + "model": "lcpp6-ornith-122b-q4kxl", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 48, + "tests_fail": 0, + "subcmds": 9, + "tps": 13.4, + "minutes": 90, + "compactions": null, + "ctx": null, + "kv": null, + "verdict": "pass", + "finished": "2026-10-23 13:49", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 1502, + "errors_per_kloc": 0.0 + } + }, { - "case": "t3o-tiel-coder-35b-q6", - "model": "tiel-coder-35b-q6", + "case": "mx22-ornith-122b-q4kxl-t2-rust-mcp-r1", + "model": "lcpp6-ornith-122b-q4kxl", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 43, + "tests_fail": 0, + "subcmds": null, + "tps": 12.9, + "minutes": 82, + "compactions": null, + "ctx": null, + "kv": null, + "verdict": "pass", + "finished": "2026-10-23 18:18", + "struggle": { + "compiles": 14, + "failed_compiles": 6, + "rust_errors": 12, + "loc": 1648, + "errors_per_kloc": 7.3 + } + }, + { + "case": "mx22-q38-defiance-192k-author-t1-cli-r1", + "model": "lcpp6-q38-defiance-192k-author", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 82, + "tests_fail": 0, + "subcmds": 13, + "tps": 32.5, + "minutes": 34, + "compactions": null, + "ctx": 196608, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 18:53", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 1944, + "errors_per_kloc": 0.0 + } + }, + { + "case": "mx22-q38-defiance-192k-author-t2-rust-mcp-r1", + "model": "lcpp6-q38-defiance-192k-author", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 46, + "tests_fail": 0, + "subcmds": null, + "tps": 22.6, + "minutes": 65, + "compactions": null, + "ctx": 196608, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 23:00", + "struggle": { + "compiles": 27, + "failed_compiles": 6, + "rust_errors": 15, + "loc": 2415, + "errors_per_kloc": 6.6 + } + } + ], + "test3": [ + { + "case": "t3g-qwen3.8-27b-vllm", + "model": "qwen3.8-27b-vllm", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "One probe session, nine numbered calls with exact arguments and fields, errors reported verbatim; built REST ground truth first, then checked the probe against REST and the shim wire log (\"the probe is an LLM and could have hallucinated\"); trusted the known-good server after a full source audit. 15 min, 49.5 tok/s, 2 compactions. Read /t3/shim.jsonl and /opt/t3/shim.py (harness visible to the agent).", + "tps": 49.5, + "minutes": 15, + "finished": "2026-10-07 19:15" + }, + { + "case": "t3o-qwen3.8-27b-vllm", + "model": "qwen3.8-27b-vllm", "flavor": "3O", "mcp_facts": "14/14", "api_facts": "14/14", "judgement_right": true, "rubric": 12, - "note": "Its first probe found the list tools failing (\"expected record, received array on structuredContent\"); after confirming it by running the binary directly it wrapped the lists in objects and moved domain and entity_id filtering client-side, then a targeted probe showed the probe cannot see structuredContent, so it put those values into the text output (fixed server 5 tools, 28 tests). A fourth probe matched all 14 facts and a fresh REST pull agreed; it drove the binary directly four times (main_server_runs=4). 28 min, 38.4 tok/s.", - "tps": 38.4, - "minutes": 28, - "finished": "2026-10-09 11:34" + "note": "Own server (5 tools) answered every fact it could correctly but had no services tool; built REST ground truth, found the gap, added ha_list_services, extended tests to 79, rebuilt; fixed server grades builds, 6 tools, 80 tests. First probe after the rebuild saw the old 5 tools (OpenCode subagent tool snapshot); it diagnosed that from the shim log and started a fresh probe naming the 6 tools. Drove the rebuilt binary directly twice as a sanity check (main_server_runs=2). 13 min, 51.2 tok/s.", + "tps": 51.2, + "minutes": 13, + "finished": "2026-10-07 19:47" }, { - "case": "t3g-cyber-tiel-coder-35b", - "model": "cyber-tiel-coder-35b", + "case": "t3g-strata-flash-next-p2", + "model": "strata-flash-next-p2", "flavor": "3G", "mcp_facts": "14/14", "api_facts": "14/14", "judgement_right": true, "rubric": 12, - "note": "It built REST ground truth first, then sent six probe sessions: a wiring check, four parallel one-tool probes with exact arguments asking for the verbatim structuredContent and errors, and a re-probe with limit 1 after the server's 24,000-character text cap cut off the 124-row unavailable reply before the \"matched\" field. All 14 facts matched; it cross-checked the instance counts against raw REST, ran the 40 tests and trusted the server, but did not find the targets_entities defect although its probe output showed \"targets_entities\":false for all three lock services. 21 min, 24.9 tok/s.", - "tps": 24.9, + "note": "One probe session with eight numbered calls naming exact arguments and fields, \"quote the error text exactly and continue\"; built REST ground truth with jq first and diffed every fact. Found a real defect in the known-good server: targets_entities checks only target.entity_id, but all 177 targeted services in this HA publish target.entity, so every service reported false. Fixed it with a test (fixed server 12 tools, 155 tests), rebuilt and re-probed. Drove the binary directly four times (main_server_runs=4). 21 min, 20.0 tok/s on two-slot Strata.", + "tps": 20.0, "minutes": 21, - "finished": "2026-10-09 11:59" + "finished": "2026-10-07 20:29" }, { - "case": "t3o-cyber-tiel-coder-35b", - "model": "cyber-tiel-coder-35b", + "case": "t3o-strata-flash-next-p2", + "model": "strata-flash-next-p2", "flavor": "3O", "mcp_facts": "14/14", "api_facts": "14/14", "judgement_right": true, "rubric": 12, - "note": "Running its own binary directly showed list_services failing on this installation's array-shaped /api/services; two parallel probes then found list_entities rejected (\"expected record, received array\" on structuredContent) and version missing from the get_config text. It wrapped the list outputs in objects with counts, accepted both catalog shapes, added version to the text and tests (fixed server 6 tools, 38 tests), ran the binary once more (main_server_runs=2) and re-probed: all 14 facts matched. 21 min, 34.4 tok/s.", - "tps": 34.4, - "minutes": 21, - "finished": "2026-10-09 12:21" + "note": "Three probe sessions: an eight-question brief with method notes (paginate, cross-check counts, quote errors), a verbatim cross-check through resources and tools, and a re-check after the fix. Found that its own ha_overview counted 'unknown' entities as 'unavailable'; split the counts, updated the schema, text resource and tests (fixed server 10 tools, 106 tests), re-probed. 17 min, 18.0 tok/s.", + "tps": 18.1, + "minutes": 17, + "finished": "2026-10-07 20:45" }, { - "case": "t3g-q38-27b-cyber", - "model": "q38-27b-cyber", + "case": "t3g-orcarouter-lcpp6", + "model": "lcpp6-orcarouter-fn-uncensored", "flavor": "3G", - "mcp_facts": "0/14", - "api_facts": "0/14", - "judgement_right": false, - "rubric": null, - "note": "Not scored: stopped by hand at 105 min (operator saw the loop). A probe session repeated the identical ha_list_entities {\"limit\":30,\"state\":\"unavailable\"} call 32+ times while writing \"Let me actually pass 300\" (audit: 13 runs of identical calls, longest 52); no facts.json. Third Q38 Cyber IQ4_XS loop; led to OpenCode doom_loop deny for every case and the Q38 mend (presence penalty rerun, author Q6_K/Q8_0). 19.4 tok/s.", - "tps": 19.4, - "minutes": 105, - "finished": "2026-10-09 14:12" + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "One probe session with the eight questions in plain terms (\"entities, of any kind, whose state is unavailable\"), asking for raw tool output, the tool and arguments used, and failures said explicitly; the probe answered in 10 calls and cross-checked counts itself. Built REST ground truth in Python, compared every fact, reviewed the handler code for those paths, and spot-checked details the probe added unasked to rule out a made-up answer. Trusted the server, correctly for these facts; did not look at list_services and so did not find the targets_entities defect the Strata run fixed (outside the rubric). Leanest run so far: 12 min, 6.1k output tokens, 8.4 tok/s.", + "tps": 8.4, + "minutes": 12, + "finished": "2026-10-07 21:04" }, { - "case": "t3g-heretic2-lcpp6", - "model": "lcpp6-flashnext-heretic2", + "case": "t3o-orcarouter-lcpp6", + "model": "lcpp6-orcarouter-fn-uncensored", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "First probe: nine plain questions asking for exact raw tool output or the exact error and the tool used. list_services came back as a client validation error (\"expected record, received array\" on structuredContent); it checked the MCP schema, confirmed structuredContent must be an object, sent a second probe aimed at the other tools that return arrays (list_events, single-key get_config), and found the same defect in all three. Wrapped them in objects, added a guard in mcp.rs, updated tests (fixed server 8 tools, 55 tests), re-probed with eleven items. REST ground truth for every fact. 46 min, 5.8 tok/s (40 MoE layers on CPU).", + "tps": 5.8, + "minutes": 46, + "finished": "2026-10-07 21:53" + }, + { + "case": "t3g-thinkingcap-heretic", + "model": "lcpp6-thinkingcap-27b-heretic", "flavor": "3G", "mcp_facts": "14/14", "api_facts": "14/14", "judgement_right": true, "rubric": 12, - "note": "One probe session with eight numbered calls naming exact arguments and fields, the tool's own matched/returned/next_cursor plus the probe's own tally, a list_domains cross-check and errors quoted verbatim; then built REST ground truth in one curl and Python pass and matched all 14 facts. Trusted the known-good server after reading three source files and running cargo test (155 pass). The brief asked for limit=500 and 200, above the server's caps of 200 and 100; the probe silently used 50 and paginated, and the agent neither noticed nor reported it accurately. 22 min, 4.7 tok/s; did not read the harness files.", - "tps": 5.5, - "minutes": 19, - "finished": "2026-10-10 02:58" + "note": "Read the whole source, built REST ground truth (caught and fixed a bug in its own script), then two probes in parallel, each a short list of exact tool calls with exact arguments, the one field to report per call, \"ABSENT\" markers for missing attributes and verbatim errors. Every value matched; trusted the server after the source read. 9 min, 19.8 tok/s, the fastest test 3 run.", + "tps": 19.8, + "minutes": 9, + "finished": "2026-10-07 22:02" }, { - "case": "t3o-heretic2-lcpp6", - "model": "lcpp6-flashnext-heretic2", + "case": "t3o-thinkingcap-heretic", + "model": "lcpp6-thinkingcap-27b-heretic", "flavor": "3O", - "mcp_facts": "5/14", + "mcp_facts": "14/14", "api_facts": "14/14", - "judgement_right": false, - "rubric": 8, - "note": "Two probe sessions with good numbered briefs (exact definitions, errors quoted verbatim, final 14-fact table) and a full REST ground truth. Its own server returns data only in structuredContent with a one-line text summary, so the probe saw just \"434 entities\", \"light: 52 entities\" etc. and nine facts stayed null; the agent blamed \"harness truncation\" without checking, fixed only the real but secondary list_services outputSchema error (services declared array, HA sends object), re-probed, still got \"1 domains\", and gave up (\"this is getting too deep\"). Declared mcp_correct=true on 5/14 delivered facts and claimed in REPORT.md to have verified the missing values from raw server output, which the transcript does not show. Drove the binary directly three times; no harness files read; no tool loops, though its reasoning after probe 2 repeats itself about five times. 45 min, 10.0 tok/s.", - "tps": 10.3, - "minutes": 44, - "finished": "2026-10-10 03:49" + "judgement_right": true, + "rubric": 12, + "note": "Baseline probe named its own tools and asked for the text and the structured value per call, errors verbatim. Found that temperature, HVAC mode and lock service names lived only in structuredContent, which OpenCode does not pass to the probe, so they were unreachable through MCP; cross-checked an odd 126 'unknown' figure against REST (correct). Fixed by putting attributes and per-domain service names in the text output with unit tests (fixed server 8 tools, 58 tests), re-probed and confirmed. 15 min, 24.7 tok/s.", + "tps": 24.7, + "minutes": 15, + "finished": "2026-10-07 22:18" }, { - "case": "t3g-abliterated-s15-lcpp6", - "model": "lcpp6-flashnext-abliterated-s15", - "flavor": "3G", + "case": "t3o-qwen3.8-27b-uncensored-vllm-pp", + "model": "qwen3.8-27b-uncensored-vllm", + "flavor": "3O", "mcp_facts": "14/14", "api_facts": "14/14", "judgement_right": true, "rubric": 12, - "note": "It built REST ground truth first, then sent one probe session with eight numbered calls giving exact arguments and the fields to report raw, pagination to the end, state-filter cross-checks and errors quoted; all 14 facts matched, a second REST pull confirmed them, and it trusted the known-good server after reading parts of two source files (no cargo test). The probe's limit=200 reply was cut by the server's 24,000-character text cap and the probe guessed \"matched 108\" from the visible rows; the agent accepted the probe's \"live state drift\" story and wrote it into REPORT.md, although its own re-poll showed stable numbers and the server never returned 108. It did not find the targets_entities defect although the probe output showed \"targets_entities\":false for all three lock services; no tool loops, no harness files read. 23 min, 5.4 tok/s.", - "tps": 6.2, - "minutes": 20, - "finished": "2026-10-10 04:20" + "note": "It measured the /api/states payload (94 KB) and split the probe work into one metadata session and three parallel per-domain scans, each returning a fixed JSON shape with verbatim errors; its own list_entities has no state filter, so the probe counted \"unavailable\" rows itself. It diffed every fact and every per-domain unavailable count against REST (no mismatches), cleared its one suspicion (the /api/entities fallback) by curl, and trusted the server without changes. 11 min, 42.3 tok/s.", + "tps": 42.3, + "minutes": 11, + "finished": "2026-10-09 09:39" }, { - "case": "t3o-abliterated-s15-lcpp6", - "model": "lcpp6-flashnext-abliterated-s15", - "flavor": "3O", + "case": "t3g-qwen3.8-27b-uncensored-vllm-pp", + "model": "qwen3.8-27b-uncensored-vllm", + "flavor": "3G", "mcp_facts": "14/14", "api_facts": "14/14", "judgement_right": true, - "rubric": 8, - "note": "Five briefs resumed in one probe session (hence 1 counted session), 93 of 128 minutes spent in the probe hand-counting 434 entities at 3.5 tok/s; built REST ground truth first and chased every mismatch (123 and 113 vs 124 unavailable, 7 vs 3 lock services), also by driving the binary directly (main_server_runs=6). Found and fixed a real defect (ha_ping, ha_get_services and ha_get_error_log returned a bare string or array in structuredContent, rejected by OpenCode with \"expected record\"; wrapped in objects, 20 tests pass) and re-probed, but after the fix OpenCode truncated the 58 KB services payload to nothing, so the probe never saw the lock services through MCP. The accepted final answer came from a round with zero MCP calls after a leading brief, and REPORT.md says the probe \"quoted verbatim\"; no tool loops, no harness files read.", - "tps": 3.5, - "minutes": 127, - "finished": "2026-10-10 06:32" + "rubric": 10, + "note": "One probe session with twelve numbered calls giving exact arguments and the fields to copy verbatim; it built REST ground truth and all 14 facts matched, then re-probed after its change. It did not find the targets_entities defect, although its own probe output showed \"targets_entities\":false for all three lock services; instead it rewrote only_available from \"drop UI-hidden entities\" to \"drop unavailable entities\", which the tool description, README and tests all documented as the intended behaviour (\"the planter's trick\"). 18 min, 47.4 tok/s; read the head of /t3/shim.jsonl and ran /opt/t3/shim.py --help (harness visible to the agent).", + "tps": 47.4, + "minutes": 18, + "finished": "2026-10-09 09:27" }, { - "case": "t3g-qwen3.8-flash-next-unpruned", - "model": "qwen3.8-flash-next-unpruned", + "case": "t3g-orcasaq2-cyber-27b", + "model": "orcasaq2-cyber-27b", "flavor": "3G", "mcp_facts": "14/14", "api_facts": "14/14", "judgement_right": true, "rubric": 12, - "note": "One probe session with eight numbered questions in plain terms (state \"exactly unavailable\", any domain), full entity lists for verification and errors with tool, arguments and exact message; the probe hit the 24,000-character text clip on the 124-row reply and worked around it with per-domain matched counts (33 distinct calls, no loop). It compared all 14 facts with REST, confirmed the missing hvac_mode attribute on the single-entity endpoint, read the handlers and ran the 155 tests, then trusted the server; it did not find the targets_entities defect, and REPORT.md wrongly calls that field verified against the API and claims an element-for-element list comparison it never ran. 67 min, 2.9 tok/s: 54% of the run was waiting for first tokens (631 s before the first output, 605 s reprocessing 36k tokens with no cache hit); no harness files read.", - "tps": 3.4, - "minutes": 56, - "finished": "2026-10-10 07:45" + "note": "It sent one probe session with the eight questions in plain terms, asking for the exact tool calls, the raw values and the exact error text; it built REST ground truth in Python, matched all 14 facts, and then read the source. It suspected that only_available filters UI-hidden entities instead of unavailable ones, but correctly kept the documented behaviour (schema and README agree) and trusted the server; it did not find the targets_entities defect, although its probe reported \"targets_entities: false\" for all three lock services. After three context compactions it had lost the prompt's exact facts.json shape and wrote its own nested per-fact shape (facts_file malformed although every value was right). 16 min, 42.7 tok/s. Regraded 2026-10-11: facts credited in the layout the model wrote (14/14 MCP, 14/14 REST); in the asked shape it was 0/14 and 0/14.", + "tps": 42.7, + "minutes": 16, + "finished": "2026-10-09 09:56" }, { - "case": "t3o-qwen3.8-flash-next-unpruned", - "model": "qwen3.8-flash-next-unpruned", + "case": "t3o-orcasaq2-cyber-27b", + "model": "orcasaq2-cyber-27b", "flavor": "3O", "mcp_facts": "14/14", "api_facts": "14/14", "judgement_right": true, "rubric": 12, - "note": "Two probe sessions: eight numbered facts in plain terms (state \"unavailable\" in ANY domain, tool used, raw output, counted entity IDs, errors with their text), then a deliberate nine-call re-check with exact arguments to rule out a made-up first report; all 37 calls are in the wire log with no errors, no loop, and no harness files read. It matched all 14 facts against REST, diffed the probe's unavailable ID lists for 8 of 10 domains, read all six source files and trusted its own server unchanged, exercising only 5 of the 9 tools. REPORT.md overstates the ID-set comparison (sensor and switch, 71 of 124, were count-only) and cites the crate's tests although cargo test timed out after 600 s unreported; 60 min, 3.0 tok/s (6.2 with probe output), 37% of the run waiting for first tokens.", - "tps": 3.0, - "minutes": 59, - "finished": "2026-10-10 08:44" + "note": "It built REST ground truth and read its own six-file server before sending one probe session: ten numbered calls with exact tool names, arguments and the field to report, errors to be quoted, and a summary table at the end. All 14 facts matched, a fresh REST pull confirmed them, and it trusted the server without changes. Two compactions left its summary with only the key names, so it wrote mcp_correct as a per-fact object rather than a single boolean. 8 min, 45.5 tok/s.", + "tps": 45.5, + "minutes": 8, + "finished": "2026-10-09 10:04" }, { - "case": "t3g-maya-glm-5.3-flash", - "model": "maya-glm-5.3-flash", + "case": "t3g-qwen3.8-27b", + "model": "qwen3.8-27b", "flavor": "3G", - "mcp_facts": "0/14", - "api_facts": "0/14", - "judgement_right": false, - "rubric": 11, - "note": "Killed at the 3-hour cap with no facts.json. The model misread and miswrote multi-digit numbers (probes reported 4342/4344 where the server sent 434; six calls with limit \"1 0\"), blamed a display channel, and spent about 2.3 hours on letter-encoding workarounds, four compactions (37,200 tokens) and one repeated 14,000-token step. Probe 3 finally matched all 14 facts and it had decided to trust the unchanged server; no harness files read; 12 tok/s.", - "tps": 11.3, - "minutes": 171, - "finished": "2026-10-10 11:45" + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "It read the whole source and built REST ground truth before one probe session with eight numbered calls giving exact arguments and fields; the probe corrected its brief's wrong definition of total_entities. It checked the probe's calls against the shim log, confirmed the binary's hash matched a fresh build, ran the 22 tests, re-pulled REST and trusted the server; it did not find the targets_entities defect and read /opt/t3/shim.py and /t3/shim.jsonl (harness visible to the agent). 16 min, 21.8 tok/s.", + "tps": 21.8, + "minutes": 16, + "finished": "2026-10-09 10:22" + }, + { + "case": "t3o-qwen3.8-27b", + "model": "qwen3.8-27b", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Its first probe matched all 14 facts; it then checked the numbers against the raw server replies in the shim log and sent a cross-check probe, which found ha_events failing (\"expected record, received array\"). It wrapped the array in {count, events}, updated the end-to-end test (fixed server 8 tools, 84 tests), re-probed and ran a final probe so every value came from the fixed binary; it read /opt/t3/shim.py and /t3/shim.jsonl (harness visible to the agent). 15 min, 19.6 tok/s.", + "tps": 19.6, + "minutes": 15, + "finished": "2026-10-09 10:38" + }, + { + "case": "t3g-tiel-coder-35b-q6", + "model": "tiel-coder-35b-q6", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "It sent one probe session with ten numbered calls giving exact arguments and the field it needed from each, asking for isError, the verbatim JSON text and a closing summary block; it built REST ground truth first, noticed that the probe's pasted state=off block was internally inconsistent and settled that, and the probe's clipped 124-entity reply (the server caps text at 24,000 characters while structuredContent stays complete), by driving the binary directly over JSON-RPC (main_server_runs=6). It trusted the server, correctly for these facts, but did not find the targets_entities defect although its probe output showed \"targets_entities\":false for all three lock services. 23 min, 22.4 tok/s.", + "tps": 22.4, + "minutes": 23, + "finished": "2026-10-09 11:05" + }, + { + "case": "t3o-tiel-coder-35b-q6", + "model": "tiel-coder-35b-q6", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Its first probe found the list tools failing (\"expected record, received array on structuredContent\"); after confirming it by running the binary directly it wrapped the lists in objects and moved domain and entity_id filtering client-side, then a targeted probe showed the probe cannot see structuredContent, so it put those values into the text output (fixed server 5 tools, 28 tests). A fourth probe matched all 14 facts and a fresh REST pull agreed; it drove the binary directly four times (main_server_runs=4). 28 min, 38.4 tok/s.", + "tps": 38.4, + "minutes": 28, + "finished": "2026-10-09 11:34" + }, + { + "case": "t3g-cyber-tiel-coder-35b", + "model": "cyber-tiel-coder-35b", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "It built REST ground truth first, then sent six probe sessions: a wiring check, four parallel one-tool probes with exact arguments asking for the verbatim structuredContent and errors, and a re-probe with limit 1 after the server's 24,000-character text cap cut off the 124-row unavailable reply before the \"matched\" field. All 14 facts matched; it cross-checked the instance counts against raw REST, ran the 40 tests and trusted the server, but did not find the targets_entities defect although its probe output showed \"targets_entities\":false for all three lock services. 21 min, 24.9 tok/s.", + "tps": 24.9, + "minutes": 21, + "finished": "2026-10-09 11:59" + }, + { + "case": "t3o-cyber-tiel-coder-35b", + "model": "cyber-tiel-coder-35b", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Running its own binary directly showed list_services failing on this installation's array-shaped /api/services; two parallel probes then found list_entities rejected (\"expected record, received array\" on structuredContent) and version missing from the get_config text. It wrapped the list outputs in objects with counts, accepted both catalog shapes, added version to the text and tests (fixed server 6 tools, 38 tests), ran the binary once more (main_server_runs=2) and re-probed: all 14 facts matched. 21 min, 34.4 tok/s.", + "tps": 34.4, + "minutes": 21, + "finished": "2026-10-09 12:21" + }, + { + "case": "t3g-q38-27b-cyber", + "model": "q38-27b-cyber", + "flavor": "3G", + "mcp_facts": "0/14", + "api_facts": "0/14", + "judgement_right": false, + "rubric": null, + "note": "Not scored: stopped by hand at 105 min (operator saw the loop). A probe session repeated the identical ha_list_entities {\"limit\":30,\"state\":\"unavailable\"} call 32+ times while writing \"Let me actually pass 300\" (audit: 13 runs of identical calls, longest 52); no facts.json. Third Q38 Cyber IQ4_XS loop; led to OpenCode doom_loop deny for every case and the Q38 mend (presence penalty rerun, author Q6_K/Q8_0). 19.4 tok/s.", + "tps": 19.4, + "minutes": 105, + "finished": "2026-10-09 14:12" + }, + { + "case": "t3g-heretic2-lcpp6", + "model": "lcpp6-flashnext-heretic2", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "One probe session with eight numbered calls naming exact arguments and fields, the tool's own matched/returned/next_cursor plus the probe's own tally, a list_domains cross-check and errors quoted verbatim; then built REST ground truth in one curl and Python pass and matched all 14 facts. Trusted the known-good server after reading three source files and running cargo test (155 pass). The brief asked for limit=500 and 200, above the server's caps of 200 and 100; the probe silently used 50 and paginated, and the agent neither noticed nor reported it accurately. 22 min, 4.7 tok/s; did not read the harness files.", + "tps": 5.5, + "minutes": 19, + "finished": "2026-10-10 02:58" + }, + { + "case": "t3o-heretic2-lcpp6", + "model": "lcpp6-flashnext-heretic2", + "flavor": "3O", + "mcp_facts": "5/14", + "api_facts": "14/14", + "judgement_right": false, + "rubric": 8, + "note": "Two probe sessions with good numbered briefs (exact definitions, errors quoted verbatim, final 14-fact table) and a full REST ground truth. Its own server returns data only in structuredContent with a one-line text summary, so the probe saw just \"434 entities\", \"light: 52 entities\" etc. and nine facts stayed null; the agent blamed \"harness truncation\" without checking, fixed only the real but secondary list_services outputSchema error (services declared array, HA sends object), re-probed, still got \"1 domains\", and gave up (\"this is getting too deep\"). Declared mcp_correct=true on 5/14 delivered facts and claimed in REPORT.md to have verified the missing values from raw server output, which the transcript does not show. Drove the binary directly three times; no harness files read; no tool loops, though its reasoning after probe 2 repeats itself about five times. 45 min, 10.0 tok/s.", + "tps": 10.3, + "minutes": 44, + "finished": "2026-10-10 03:49" + }, + { + "case": "t3g-abliterated-s15-lcpp6", + "model": "lcpp6-flashnext-abliterated-s15", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "It built REST ground truth first, then sent one probe session with eight numbered calls giving exact arguments and the fields to report raw, pagination to the end, state-filter cross-checks and errors quoted; all 14 facts matched, a second REST pull confirmed them, and it trusted the known-good server after reading parts of two source files (no cargo test). The probe's limit=200 reply was cut by the server's 24,000-character text cap and the probe guessed \"matched 108\" from the visible rows; the agent accepted the probe's \"live state drift\" story and wrote it into REPORT.md, although its own re-poll showed stable numbers and the server never returned 108. It did not find the targets_entities defect although the probe output showed \"targets_entities\":false for all three lock services; no tool loops, no harness files read. 23 min, 5.4 tok/s.", + "tps": 6.2, + "minutes": 20, + "finished": "2026-10-10 04:20" + }, + { + "case": "t3o-abliterated-s15-lcpp6", + "model": "lcpp6-flashnext-abliterated-s15", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 8, + "note": "Five briefs resumed in one probe session (hence 1 counted session), 93 of 128 minutes spent in the probe hand-counting 434 entities at 3.5 tok/s; built REST ground truth first and chased every mismatch (123 and 113 vs 124 unavailable, 7 vs 3 lock services), also by driving the binary directly (main_server_runs=6). Found and fixed a real defect (ha_ping, ha_get_services and ha_get_error_log returned a bare string or array in structuredContent, rejected by OpenCode with \"expected record\"; wrapped in objects, 20 tests pass) and re-probed, but after the fix OpenCode truncated the 58 KB services payload to nothing, so the probe never saw the lock services through MCP. The accepted final answer came from a round with zero MCP calls after a leading brief, and REPORT.md says the probe \"quoted verbatim\"; no tool loops, no harness files read.", + "tps": 3.5, + "minutes": 127, + "finished": "2026-10-10 06:32" + }, + { + "case": "t3g-qwen3.8-flash-next-unpruned", + "model": "qwen3.8-flash-next-unpruned", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "One probe session with eight numbered questions in plain terms (state \"exactly unavailable\", any domain), full entity lists for verification and errors with tool, arguments and exact message; the probe hit the 24,000-character text clip on the 124-row reply and worked around it with per-domain matched counts (33 distinct calls, no loop). It compared all 14 facts with REST, confirmed the missing hvac_mode attribute on the single-entity endpoint, read the handlers and ran the 155 tests, then trusted the server; it did not find the targets_entities defect, and REPORT.md wrongly calls that field verified against the API and claims an element-for-element list comparison it never ran. 67 min, 2.9 tok/s: 54% of the run was waiting for first tokens (631 s before the first output, 605 s reprocessing 36k tokens with no cache hit); no harness files read.", + "tps": 3.4, + "minutes": 56, + "finished": "2026-10-10 07:45" + }, + { + "case": "t3o-qwen3.8-flash-next-unpruned", + "model": "qwen3.8-flash-next-unpruned", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Two probe sessions: eight numbered facts in plain terms (state \"unavailable\" in ANY domain, tool used, raw output, counted entity IDs, errors with their text), then a deliberate nine-call re-check with exact arguments to rule out a made-up first report; all 37 calls are in the wire log with no errors, no loop, and no harness files read. It matched all 14 facts against REST, diffed the probe's unavailable ID lists for 8 of 10 domains, read all six source files and trusted its own server unchanged, exercising only 5 of the 9 tools. REPORT.md overstates the ID-set comparison (sensor and switch, 71 of 124, were count-only) and cites the crate's tests although cargo test timed out after 600 s unreported; 60 min, 3.0 tok/s (6.2 with probe output), 37% of the run waiting for first tokens.", + "tps": 3.0, + "minutes": 59, + "finished": "2026-10-10 08:44" + }, + { + "case": "t3g-maya-glm-5.3-flash", + "model": "maya-glm-5.3-flash", + "flavor": "3G", + "mcp_facts": "0/14", + "api_facts": "0/14", + "judgement_right": false, + "rubric": 11, + "note": "Killed at the 3-hour cap with no facts.json. The model misread and miswrote multi-digit numbers (probes reported 4342/4344 where the server sent 434; six calls with limit \"1 0\"), blamed a display channel, and spent about 2.3 hours on letter-encoding workarounds, four compactions (37,200 tokens) and one repeated 14,000-token step. Probe 3 finally matched all 14 facts and it had decided to trust the unchanged server; no harness files read; 12 tok/s.", + "tps": 11.3, + "minutes": 171, + "finished": "2026-10-10 11:45" }, { "case": "t3o-maya-glm-5.3-flash", @@ -8093,6 +8718,32 @@ "tps": 7.1, "minutes": 9, "finished": "2026-10-17 20:13" + }, + { + "case": "mx22-flashnext-heretic2-setupretry1-t3g-known-good-mcp-r1", + "model": "lcpp6-flashnext-heretic2", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Known-good server. One probe, 12 calls. REST and MCP both 14 of 14. It left the server unchanged and the judgement is right. Rubric 12.", + "tps": 9.5, + "minutes": 15, + "finished": "2026-10-23 11:39" + }, + { + "case": "mx22-flashnext-heretic2-setupretry1-t3o-own-mcp-r1", + "model": "lcpp6-flashnext-heretic2", + "flavor": "3O", + "mcp_facts": "13/14", + "api_facts": "14/14", + "judgement_right": false, + "rubric": 9, + "note": "Own server. MCP 13 of 14, REST 14 of 14. The fixed server builds, initializes, exposes 7 tools and passes 34 tests, but temperature was still missed and the final judgement is wrong. Rubric 9.", + "tps": 11.8, + "minutes": 32, + "finished": "2026-10-23 12:11" } ], "ab": [ @@ -8357,104 +9008,2009 @@ "error": "" }, { - "model": "strata44-flash-next-pcie", - "engine": "Strata 0.1.44.1", - "variant": "2 rounds", - "decode": 44.9, - "quote": null, - "prefill": 1416.4, - "vram": null, - "error": "" + "model": "strata44-flash-next-pcie", + "engine": "Strata 0.1.44.1", + "variant": "2 rounds", + "decode": 44.9, + "quote": null, + "prefill": 1416.4, + "vram": null, + "error": "" + }, + { + "model": "maya-glm-5.3-flash", + "engine": "Maya 1.0.4", + "variant": "2 rounds, 3.5 bpw Maya-S", + "decode": 14.6, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "maya1111-glm-5.3-flash", + "engine": "Maya 1.1.11", + "variant": "2 rounds, same pack", + "decode": 17.0, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "exl3-qwen3.8-27b-dflash2", + "engine": "exllamav3 1.6.1", + "variant": "2 rounds, DFlash2, thinking off", + "decode": 138.6, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "exl3-qwen3.8-27b-dflash2-162", + "engine": "exllamav3 1.6.2", + "variant": "2 rounds, DFlash2, thinking off", + "decode": 140.1, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "strata020-gyro-s-mtp-test", + "engine": "Strata 0.2.0-beta", + "variant": "Gyro-S + MTP, code prompt", + "decode": 118.6, + "quote": null, + "prefill": 2550.9, + "vram": 23862, + "error": "" + }, + { + "model": "strata44-flash-next-pcie", + "engine": "Strata 0.2.0-beta", + "variant": "IQ3_S + MTP, code prompt", + "decode": 35.4, + "quote": null, + "prefill": 2645.4, + "vram": 23868, + "error": "" + }, + { + "model": "strata44-flash-next-pcie", + "engine": "Strata 0.1.45", + "variant": "IQ3_S + MTP, code prompt", + "decode": 29.4, + "quote": null, + "prefill": 2597.8, + "vram": 23874, + "error": "" + }, + { + "model": "strata020-gyro-s-mtp-test", + "engine": "Strata 0.2.2", + "variant": "Gyro-S + MTP, code prompt", + "decode": 144.1, + "quote": null, + "prefill": 2545.7, + "vram": 23862, + "error": "" + }, + { + "model": "strata020-gyro-s-mtp-test", + "engine": "Strata 0.2.0-beta", + "variant": "Gyro-S + MTP, code prompt, same-day control", + "decode": 121.6, + "quote": null, + "prefill": 2539.8, + "vram": 23862, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock off, 400-token code prompt", + "decode": 126.4, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock off, 140826-token prompt, seed 41", + "decode": 85.3, + "quote": null, + "prefill": 736, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock off, 140826-token prompt, seed 42", + "decode": 79.1, + "quote": null, + "prefill": 736, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 1, 400-token code prompt", + "decode": 127.9, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 1, 140826-token prompt, seed 41", + "decode": 94.5, + "quote": null, + "prefill": 1053, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 1, 140826-token prompt, seed 42", + "decode": 87.9, + "quote": null, + "prefill": 1053, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 2, 400-token code prompt", + "decode": 127.5, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 2, 140826-token prompt, seed 41", + "decode": 94.4, + "quote": null, + "prefill": 1056, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 2, 140826-token prompt, seed 42", + "decode": 87.7, + "quote": null, + "prefill": 1055, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock off, 400-token code prompt", + "decode": 149.1, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock off, 141004-token prompt, seed 41", + "decode": 97.7, + "quote": null, + "prefill": 2201, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock off, 141004-token prompt, seed 42", + "decode": 97.6, + "quote": null, + "prefill": 1549, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 1, 400-token code prompt", + "decode": 148.6, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 1, 141004-token prompt, seed 41", + "decode": 97.7, + "quote": null, + "prefill": 2397, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 1, 141004-token prompt, seed 42", + "decode": 97.9, + "quote": null, + "prefill": 1746, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 2, 400-token code prompt", + "decode": 148.8, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 2, 141004-token prompt, seed 41", + "decode": 97.7, + "quote": null, + "prefill": 2396, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 2, 141004-token prompt, seed 42", + "decode": 97.5, + "quote": null, + "prefill": 1745, + "vram": null, + "error": "" + } + ], + "engines": [ + { + "id": "llama-cpp", + "name": "llama.cpp", + "kind": "GGUF server; dense models and MoE with experts in RAM", + "versions": [ + { + "version": "0.4.1-dev (commit 4260903)", + "date": "2026-09-23 to 2026-10-18", + "note": "Bundled build used by the native entries; Flash-Next with experts in RAM: 2.2 tok/s on the probe; long agent runs on it ran out of time." + }, + { + "version": "0.6.0 (official image, build 11429)", + "date": "2026-10-05", + "note": "2.5-4x faster than 0.4 on Flash-Next with experts in RAM; MTP and n-gram drafting fixes." + }, + { + "version": "0.7.0", + "date": "2026-10-18", + "note": "Released 2026-10-15. Adds a GPU cache for experts kept in RAM (--moe-cache-mib); in the router since 2026-10-18." + }, + { + "version": "Newer official builds", + "date": "2026-10-21", + "note": "Flint needs b11803 or later; Carbon-X needs b11742 or later; trials ran by hand, not in the router." + } + ], + "good_at": "Any GGUF, including MoE models with experts in system RAM", + "watch_out": "Check the build supports the model's architecture before reading any result", + "multi_session": { + "ok": true, + "note": "11 of 11 probe checks on Flash-Next entries" + }, + "speeds": [ + { + "model": "Tiel Coder 35B IQ4_XS", + "config": "0.7.0, all on GPU", + "decode": 151, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-18" + }, + { + "model": "Flint 40B-A2.5B Q3_K_XL", + "config": "newer build, all on GPU, q8 KV, short prompt", + "decode": 129, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flint 40B-A2.5B Q3_K_XL", + "config": "newer build, all on GPU, q8 KV, 11.7k depth", + "decode": 106, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flint Q4_K_XL", + "config": "10 expert layers in RAM", + "decode": 95, + "prefill": null, + "unit": "tok/s", + "context": "150k", + "date": "2026-10" + }, + { + "model": "Flint Q4_K_XL", + "config": "15 expert layers in RAM; prefill of an 11.7k prompt", + "decode": 81.4, + "prefill": 2208, + "unit": "tok/s", + "context": null, + "date": "2026-10-22" + }, + { + "model": "Flint Q4_K_XL", + "config": "19 expert layers in RAM", + "decode": 73.1, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-22" + }, + { + "model": "Flint Q4_K_XL", + "config": "24 expert layers in RAM; prefill of an 11.7k prompt", + "decode": 64.2, + "prefill": 1624, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Ornith-1.5-122B UD-Q4_K_XL", + "config": "40 expert layers on CPU", + "decode": 16.3, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-20" + }, + { + "model": "Carbon-X 14.8B Q8_0", + "config": "all on GPU", + "decode": 52, + "prefill": null, + "unit": "tok/s", + "context": "40k", + "date": "2026-10-22" + }, + { + "model": "Qwen3.8-27B", + "config": "MTP drafting, all on GPU, code", + "decode": 93, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B", + "config": "DFlash2 drafter (GGUF), code; 47 % of drafts accepted", + "decode": 95, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B", + "config": "DFlash2 drafter (GGUF), prose", + "decode": 43, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B", + "config": "DFlash2 drafter (GGUF), structured output", + "decode": 108, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-13" + }, + { + "model": "Flash-Next Orcarouter IQ4_XS", + "config": "0.7.0, -cmoe, cache 12800 MiB, q8 KV, streamed (about)", + "decode": 19, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next Heretic2 IQ4XS-NGQ4", + "config": "0.7.0, -cmoe, cache 12800 MiB, q8 KV, streamed (about)", + "decode": 18, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next Abliterated s1.5 IQ4_XS", + "config": "0.7.0, -cmoe, cache 13824 MiB, q8 KV, streamed", + "decode": "19-21", + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next Swift 1.5 IQ3_XXS (pruned)", + "config": "0.7.0, -cmoe, cache 9216 MiB, q8 KV, streamed", + "decode": "23-26", + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next unpruned IQ3_S", + "config": "0.7.0, -cmoe, cache 9216 MiB, q8 KV, streamed", + "decode": "21-23", + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next GGUFs (fresh loads)", + "config": "0.7.0 expert cache; prefill about 250 before", + "decode": null, + "prefill": "415-427", + "unit": "tok/s", + "context": "16k and 48k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next unpruned IQ3_S", + "config": "0.7.0, raw short outputs, thinking off, code; prose 41.3, structured 41.0", + "decode": 40.3, + "prefill": null, + "unit": "tok/s", + "context": "32k", + "date": "2026-10-22" + }, + { + "model": "Qwen3.8-Flash-Next", + "config": "0.4.1-dev, experts in RAM, short probe", + "decode": 2.2, + "prefill": 136, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + }, + { + "model": "Qwen3.8-Flash-Next", + "config": "0.6, experts in RAM, short probe", + "decode": 8.9, + "prefill": 569, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + }, + { + "model": "Orcarouter Flash-Next Uncensored", + "config": "0.4.1-dev, experts in RAM, short probe", + "decode": 5.3, + "prefill": 295, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + }, + { + "model": "Orcarouter Flash-Next Uncensored", + "config": "0.6, experts in RAM, short probe", + "decode": 13.1, + "prefill": 486, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + }, + { + "model": "Qwen3.8-27B Cyber", + "config": "0.6, all on GPU, no drafting", + "decode": 45.1, + "prefill": null, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + }, + { + "model": "Qwen3.8-27B Cyber", + "config": "0.6, all on GPU, with MTP", + "decode": 82.2, + "prefill": null, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + }, + { + "model": "OrcaSAQ-2-Cyber-27B", + "config": "0.6, all on GPU, no drafting", + "decode": 83.2, + "prefill": null, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + }, + { + "model": "OrcaSAQ-2-Cyber-27B", + "config": "0.6, n-gram drafts, copy-heavy output", + "decode": 412, + "prefill": null, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + }, + { + "model": "Cyber-Tiel-Coder-35B", + "config": "0.6, 20 expert layers in RAM, short probe", + "decode": 50.5, + "prefill": null, + "unit": "tok/s", + "context": "49k", + "date": "2026-10-05" + } + ], + "bugs": [ + { + "text": "0.4.1-dev ran Flash-Next with experts in RAM 2.5-4x slower than 0.6", + "status": "fixed", + "date": "2026-10-05" + }, + { + "text": "0.6: automatic -fit aborts when -ngl is set by hand", + "status": "worked around", + "date": null + }, + { + "text": "0.6: offloaded models stall on the first request without -lm none", + "status": "worked around", + "date": "2026-09-30" + }, + { + "text": "0.7.0: OrcaSAQ-2 with DFlash2 drafts failed every request on 580 MiB buffer", + "status": "worked around", + "date": "2026-10-18" + }, + { + "text": "Qwen3.8-27B Cyber on 0.7.0: empty 32,000-token replies once, unrepeated", + "status": "open", + "date": null + }, + { + "text": "Lazy tensor reads slow the first pass over unseen text when cache is cold", + "status": "open", + "date": null + }, + { + "text": "Newer-architecture GGUFs load on older builds, then fail or tokenize wrongly", + "status": "worked around", + "date": "2026-10-21" + } + ], + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/engines.md#llamacpp" + }, + { + "id": "vllm", + "name": "vLLM (HyperQwen)", + "kind": "dense model server with prefix caching", + "versions": [ + { + "version": "HyperQwen 0.30", + "date": null, + "note": "In use; 2-4x faster than llama.cpp over whole agent runs on Qwen3.8-27B." + }, + { + "version": "0.31", + "date": "2026-10-14", + "note": "Same speed rounds, slightly slower deep (79.5 against 84.5 tok/s at 141k); not adopted." + }, + { + "version": "0.32", + "date": null, + "note": "Image built and staged; not in use." + } + ], + "good_at": "Dense models on the GPU; long agent runs with prefix caching", + "watch_out": "Needs per-request thinking budget; images only with VISION=1", + "multi_session": { + "ok": true, + "note": "Main agent plus sub-agents ran cleanly, tests 3 and 8" + }, + "speeds": [ + { + "model": "Qwen3.8-27B", + "config": "MTP drafts, FP8 KV, average over a run (about)", + "decode": 70, + "prefill": null, + "unit": "tok/s", + "context": "150k", + "date": null + }, + { + "model": "Qwen3.8-27B", + "config": "MTP drafts, FP8 KV, short code", + "decode": 127, + "prefill": null, + "unit": "tok/s", + "context": "150k", + "date": null + }, + { + "model": "Qwen3.8-27B", + "config": "MTP drafts, FP8 KV, 141k of history", + "decode": 85, + "prefill": null, + "unit": "tok/s", + "context": "141k history", + "date": null + }, + { + "model": "Qwen3.8-27B", + "config": "DFlash2 drafts, 4/2-bit KV, about half the speed under load", + "decode": null, + "prefill": null, + "unit": "tok/s", + "context": "240k", + "date": null + }, + { + "model": "Qwen3.8-27B", + "config": "DFlash2 drafts (about)", + "decode": 97, + "prefill": null, + "unit": "tok/s", + "context": "64k", + "date": "2026-10-06" + }, + { + "model": "Qwen3.8-27B", + "config": "DFlash2 15-token drafts (about)", + "decode": 95, + "prefill": null, + "unit": "tok/s", + "context": "57k", + "date": "2026-10-06" + } + ], + "bugs": [ + { + "text": "With thinking on, Qwen3.8-27B finished no one-shot pagoda in seven tries", + "status": "open", + "date": "2026-10-14" + }, + { + "text": "Gemma 4 26B's server passed the grader; a real client could not connect", + "status": "open", + "date": "2026-10-10" + }, + { + "text": "DFlash2 15-token drafts at 57k: one test compacted 55 times", + "status": "worked around", + "date": "2026-10-06" + }, + { + "text": "DFlash2 at 64k: agent compacts constantly; one test overflowed the context", + "status": "worked around", + "date": "2026-10-06" + } + ], + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/engines.md#vllm" + }, + { + "id": "sglang", + "name": "SGLang (EXL3)", + "kind": "MoE server with GPU expert cache, EXL3 weights", + "versions": [ + { + "version": "Original fork build", + "date": "2026-10-01 to 2026-10-04", + "note": "200k, fp8 KV; built the best test-2 server of all runs: 12 tools, 155 tests, 124 minutes." + }, + { + "version": "Newer fork build (phase D)", + "date": "2026-10-11", + "note": "11 of 11 multi-session checks; test 2 cut off and never graded; dropped 2026-10-21." + }, + { + "version": "Three stock builds", + "date": "2026-10-21", + "note": "204,800 context, two slots, EXL3 3.05 bpw, 20.8-21.4 GB, load 252-262 s; needles at 100k, 150k, 190k passed on two of three builds (8-bit KV build not run)." + } + ], + "good_at": "Most capable Flash-Next setup tested: images, 200k window", + "watch_out": "Slower than the 27B stacks; no agent test rerun on stock builds", + "multi_session": { + "ok": true, + "note": "Stock builds: 11 of 11 checks, two slots" + }, + "speeds": [ + { + "model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", + "config": "original fork build, fp8 KV, average over the test-2 run", + "decode": 36, + "prefill": null, + "unit": "tok/s", + "context": "200k", + "date": "2026-10-04" + }, + { + "model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", + "config": "original fork build, test 1", + "decode": 44.3, + "prefill": null, + "unit": "tok/s", + "context": "200k", + "date": "2026-10-04" + }, + { + "model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", + "config": "newer fork build, test 1, earlier settings", + "decode": 25.3, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-11" + }, + { + "model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", + "config": "stock build 1, two slots, prefill at 190k", + "decode": 24.1, + "prefill": 2600, + "unit": "tok/s", + "context": "204,800", + "date": "2026-10-21" + }, + { + "model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", + "config": "stock build 2, two slots, prefill at 190k", + "decode": 24.6, + "prefill": 2600, + "unit": "tok/s", + "context": "204,800", + "date": "2026-10-21" + }, + { + "model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", + "config": "stock build 3, two slots, prefill at 190k", + "decode": 25.1, + "prefill": 2600, + "unit": "tok/s", + "context": "204,800", + "date": "2026-10-21" + } + ], + "bugs": [ + { + "text": "Original build served one session at a time and crashed under several", + "status": "fixed", + "date": "2026-10-03" + }, + { + "text": "A test-3 run on the original build needed Strata instead", + "status": "worked around", + "date": null + } + ], + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/engines.md#sglang" + }, + { + "id": "strata", + "name": "Strata", + "kind": "MoE server with hot-expert VRAM cache and built-in MTP", + "versions": [ + { + "version": "0.1.39 / 0.1.40.3", + "date": "2026-10-04 to 2026-10-09", + "note": "30-45 tok/s decode, 21-30 over a whole run; test 2 full passes with Flash-Next and Swift 1.5." + }, + { + "version": "0.1.42", + "date": "2026-10-10 to 2026-10-12", + "note": "Release defaults slower; the old PCIe share doubled long-answer decode." + }, + { + "version": "0.1.44.1", + "date": "2026-10-15", + "note": "Load 60 s against 214 s; decode 44.9 against 18.3 on 0.1.42 (mean of two rounds: cold 13.1, warm 23.5)." + }, + { + "version": "0.1.45", + "date": "2026-10-20", + "note": "IQ3_S with MTP, code prompt: 29.4 tok/s." + }, + { + "version": "0.2.0-beta", + "date": "2026-10-20", + "note": "Gyro-S with MTP: test 2 full pass in 24 minutes, 89.8 tok/s over the run; agent pagoda 9 of 10." + }, + { + "version": "0.2.2", + "date": "2026-10-22", + "note": "Gyro-S: code 121.6 to 144.1 tok/s on the same-day control, load 23 s; no agent run yet." + } + ], + "good_at": "Fastest Flash-Next decode measured; made 1-bit-class Gyro-S usable", + "watch_out": "0.2 is beta; warm up before timing; set reasoning budget", + "multi_session": { + "ok": true, + "note": "Works with parallel 2 and --batch-mtp; Gyro build unprobed" + }, + "speeds": [ + { + "model": "Qwen3.8-Flash-Next (GSQ-RCO quants)", + "config": "0.1.39 / 0.1.40.3, decode; 21-30 over a whole run", + "decode": "30-45", + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-09" + }, + { + "model": "Qwen3.8-Flash-Next", + "config": "0.1.42 release-default PCIe share, long answers", + "decode": 48, + "prefill": 1177, + "unit": "tok/s", + "context": null, + "date": "2026-10-11" + }, + { + "model": "Qwen3.8-Flash-Next", + "config": "0.1.42 old PCIe share, long answers", + "decode": 95, + "prefill": 1983, + "unit": "tok/s", + "context": null, + "date": "2026-10-11" + }, + { + "model": "Qwen3.8-Flash-Next", + "config": "0.1.42, first round, cold", + "decode": 13.1, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-15" + }, + { + "model": "Qwen3.8-Flash-Next", + "config": "0.1.42, warm round", + "decode": 23.5, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-15" + }, + { + "model": "Qwen3.8-Flash-Next", + "config": "0.1.44.1", + "decode": 44.9, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-15" + }, + { + "model": "Flash-Next IQ3_S", + "config": "0.1.45, MTP, code prompt", + "decode": 29.4, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-20" + }, + { + "model": "Flash-Next IQ3_S", + "config": "0.2.0-beta, MTP, code prompt", + "decode": 35.4, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-20" + }, + { + "model": "Flash-Next Gyro-S (TQ1_0)", + "config": "0.2.0-beta, MTP draft, code, text only", + "decode": 118.6, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-20" + }, + { + "model": "Flash-Next Gyro-S (TQ1_0)", + "config": "0.2.0-beta, MTP, average over test-2 run", + "decode": 89.8, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-20" + }, + { + "model": "Flash-Next Gyro-S (TQ1_0)", + "config": "0.2.2, MTP, code", + "decode": 144.1, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-22" + }, + { + "model": "Flash-Next Gyro-S (TQ1_0)", + "config": "0.2.2, MTP, prose", + "decode": 123.4, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-22" + }, + { + "model": "Flash-Next Gyro-S (TQ1_0)", + "config": "0.2.2, MTP, JSON", + "decode": 157.7, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-22" + } + ], + "bugs": [ + { + "text": "0.1.42 release-default PCIe share halved decode", + "status": "worked around", + "date": "2026-10-11" + }, + { + "text": "0.1.42 test 2T first start failed: engine exits locking about 40 GB host RAM", + "status": "open", + "date": null + }, + { + "text": "Gyro no-MTP arms fail to start; native pack requires speculative decoding", + "status": "open", + "date": null + }, + { + "text": "1.6-bit coder looped until reasoning_budget_tokens was set", + "status": "worked around", + "date": null + }, + { + "text": "DNS-rebinding guard returns 403 to any host name not in allowed_hosts", + "status": "worked around", + "date": null + } + ], + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/engines.md#strata" + }, + { + "id": "maya", + "name": "Maya", + "kind": "tiered-expert MoE server (VRAM, RAM, SSD)", + "versions": [ + { + "version": "1.0.4 with three patches", + "date": "2026-10-08", + "note": "About 4 tok/s out of the box; 17-19 with patches once weights were on local disk. Test 1 pass (44 tests), test 2 pass (7 tools, 66 tests, 322 minutes)." + }, + { + "version": "1.1.11", + "date": "2026-10-16", + "note": "Patches upstream; 17.0 against 14.6 tok/s, first token 18 s against 92 s, 12 of 12 digit strings." + }, + { + "version": "1.1.13", + "date": "2026-10-17", + "note": "Maya-S and Maya-M IQ2_S: 12 of 12 digits, 2 of 2 needles; test 3 rubric 12 of 12 on both." + } + ], + "good_at": "Runs a 321B MoE on one card and passed both tests", + "watch_out": "Slowest model to pass both tests; first load reads 100 GB", + "multi_session": { + "ok": null, + "note": "Not measured; treated as one request at a time" + }, + "speeds": [ + { + "model": "GLM-5.3-Flash Maya-S IQ2_XXS", + "config": "1.0.4 out of the box", + "decode": 4, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-08" + }, + { + "model": "GLM-5.3-Flash Maya-S IQ2_XXS", + "config": "1.0.4 with three patches, weights on local disk", + "decode": "17-19", + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-08" + }, + { + "model": "GLM-5.3-Flash Maya-S IQ2_XXS", + "config": "1.1.11, two probe rounds (1.0.x: 14.6 on the same 3.5 bpw pack)", + "decode": 17.0, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-16" + }, + { + "model": "GLM-5.3-Flash Maya-S / Maya-M", + "config": "1.1.13, test 3 runs, both flavors", + "decode": "5.4-6.9", + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-17" + }, + { + "model": "GLM-5.3-Flash Maya-M IQ2_S", + "config": "default PCIe share 0.36 (0.25 gave 17.9)", + "decode": 16.0, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-17" + }, + { + "model": "GLM-5.3-Flash Maya-M IQ2_S", + "config": "PCIe share 0.25 (candidate, not applied)", + "decode": 17.9, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-17" + }, + { + "model": "GLM-5.3-Flash Maya-M IQ2_S", + "config": "no DFlash2 draft head", + "decode": 15.35, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-17" + }, + { + "model": "GLM-5.3-Flash Maya-M IQ2_S", + "config": "with DFlash2 draft head, 91 % of drafts accepted", + "decode": 10.7, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-17" + } + ], + "bugs": [ + { + "text": "1.0.4 decoded about 4 tok/s; every VRAM eviction re-read from the network", + "status": "fixed", + "date": "2026-10-08" + }, + { + "text": "On 1.0.x, GLM split multi-digit tool-argument numbers with spaces", + "status": "fixed", + "date": "2026-10-16" + }, + { + "text": "DFlash2 draft on Maya-M slowed decode to 10.7 from 15.35 tok/s", + "status": "worked around", + "date": "2026-10-17" + }, + { + "text": "1-bit Huihui variant failed both tests with repetition", + "status": "open", + "date": null + } + ], + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/engines.md#maya" + }, + { + "id": "exllamav3", + "name": "ExLlamaV3 (TabbyAPI)", + "kind": "dense model server, EXL3 weights", + "versions": [ + { + "version": "1.6.1", + "date": "2026-10-13", + "note": "Runs Qwen3.8-27B from EXL3 4.00 bpw, unpatched, with its MTP head or the DFlash2 drafter." + }, + { + "version": "1.6.2", + "date": "2026-10-16", + "note": "140.1 against 138.6 tok/s on 1.6.1 with DFlash2, within noise." + }, + { + "version": "1.6.3", + "date": null, + "note": "Image built and staged; not in use." + } + ], + "good_at": "Larger window than vLLM for the 27B; DFlash2 without a patch", + "watch_out": "Slower than vLLM with MTP; drafting alters temperature-0 replies", + "multi_session": { + "ok": null, + "note": "Not measured" + }, + "speeds": [ + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "DFlash2 drafter, code; 98k window fits with 8-bit cache", + "decode": 158, + "prefill": null, + "unit": "tok/s", + "context": "16k", + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "DFlash2 drafter, prose", + "decode": 73, + "prefill": null, + "unit": "tok/s", + "context": "16k", + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "DFlash2 drafter, structured output", + "decode": 181, + "prefill": null, + "unit": "tok/s", + "context": "16k", + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "own MTP head, code; 196k window fits with 8-bit cache", + "decode": 116, + "prefill": null, + "unit": "tok/s", + "context": "16k", + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "own MTP head, prose", + "decode": 70, + "prefill": null, + "unit": "tok/s", + "context": "16k", + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "own MTP head, structured output", + "decode": 128, + "prefill": null, + "unit": "tok/s", + "context": "16k", + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "no drafting, one round", + "decode": 42, + "prefill": null, + "unit": "tok/s", + "context": "16k", + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "1.6.2, DFlash2, thinking off", + "decode": 140.1, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-16" + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "MTP served at 196k, short code; prompts read at 650-740", + "decode": 115, + "prefill": "650-740", + "unit": "tok/s", + "context": "196k", + "date": null + }, + { + "model": "Qwen3.8-27B (EXL3 4.00 bpw)", + "config": "MTP served at 196k, 141k of context", + "decode": 72, + "prefill": null, + "unit": "tok/s", + "context": "141k history", + "date": null + } + ], + "bugs": [ + { + "text": "At temperature 0, drafting changed 7 of 15 DFlash2 replies", + "status": "open", + "date": "2026-10-13" + }, + { + "text": "At temperature 0, drafting changed 10 of 15 MTP replies", + "status": "open", + "date": "2026-10-13" + }, + { + "text": "MTP setup scored 0 in five of six one-shot pagoda tries", + "status": "open", + "date": "2026-10-14" + } + ], + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/engines.md#exllamav3-tabbyapi" + }, + { + "id": "tensorfold", + "name": "TensorFold", + "kind": "dense model server with DFlash2 drafting", + "versions": [ + { + "version": "0.6.6 (Python engine)", + "date": "2026-10-13", + "note": "Fastest writer measured for Qwen3.8-27B on the 3090; Python line frozen upstream." + }, + { + "version": "1.1.0 (upstream)", + "date": "2026-10-18", + "note": "Reports native RTX 30 support; we have not built or run it." + } + ], + "good_at": "Raw decode speed; exact replies at temperature 0", + "watch_out": "Needs a local patch below compute capability 8.9; bf16 cache only", + "multi_session": { + "ok": null, + "note": "Not measured" + }, + "speeds": [ + { + "model": "Qwen3.8-27B", + "config": "DFlash2 drafter, short code (167-170 across runs)", + "decode": 170, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B", + "config": "DFlash2 drafter, prose (81-84 across runs)", + "decode": 84, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-13" + }, + { + "model": "Qwen3.8-27B", + "config": "DFlash2 drafter, structured output (209-211 across runs)", + "decode": 211, + "prefill": null, + "unit": "tok/s", + "context": null, + "date": "2026-10-13" + } + ], + "bugs": [ + { + "text": "Needed a local patch to run below compute capability 8.9", + "status": "worked around", + "date": null + }, + { + "text": "Cache is bf16 only: 42k window by default, 73k at the edge", + "status": "open", + "date": null + }, + { + "text": "Python line is frozen upstream; engine parked, not in the router", + "status": "open", + "date": null + } + ], + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/engines.md#tensorfold" + }, + { + "id": "tqllm", + "name": "tqLLM", + "kind": "Flash-Next GGUF server (trial)", + "versions": [ + { + "version": "1.2.3", + "date": "2026-10-22", + "note": "Trial with the Flash-Next IQ3_S GGUF and its MTP head against the live llama.cpp entry; first load 23 minutes cold, then 2-3." + } + ], + "good_at": "Fast short replies: about 2.5 times llama.cpp on the same GGUF", + "watch_out": "Fails after one out-of-memory prefill; health check still says ok", + "multi_session": { + "ok": false, + "note": "Failed: 4 of 11 at best; llama.cpp passed 11" + }, + "speeds": [ + { + "model": "Flash-Next IQ3_S + MTP head", + "config": "short outputs, thinking off, code; prefill about 2,000 on a 30k prompt", + "decode": 101.7, + "prefill": 2000, + "unit": "tok/s", + "context": "32k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next IQ3_S + MTP head", + "config": "short outputs, thinking off, prose", + "decode": 79.9, + "prefill": null, + "unit": "tok/s", + "context": "32k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next IQ3_S + MTP head", + "config": "short outputs, thinking off, structured output", + "decode": 111.8, + "prefill": null, + "unit": "tok/s", + "context": "32k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next IQ3_S + MTP head", + "config": "default 262k window, same bench, code", + "decode": 105.2, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next IQ3_S + MTP head", + "config": "default 262k window, same bench, prose", + "decode": 83.5, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next IQ3_S + MTP head", + "config": "default 262k window, same bench, structured output", + "decode": 107.1, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + }, + { + "model": "Flash-Next IQ3_S + MTP head", + "config": "smoke decode of a 37-token prompt, best 262k arm (not a bench)", + "decode": 53.9, + "prefill": null, + "unit": "tok/s", + "context": "262k", + "date": "2026-10-22" + } + ], + "bugs": [ + { + "text": "First prompt short on VRAM for prefill fails; every later request fails", + "status": "open", + "date": "2026-10-22" + }, + { + "text": "At 131072 the first failure came on a 289-token prompt", + "status": "open", + "date": "2026-10-22" + }, + { + "text": "Health check keeps answering ok after failure, so a router never notices", + "status": "open", + "date": "2026-10-22" + }, + { + "text": "Default 262k arm answered long prompts with HTTP 500", + "status": "open", + "date": "2026-10-22" + }, + { + "text": "Expert-cache flag had no visible effect on Flash-Next", + "status": "open", + "date": "2026-10-22" + } + ], + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/engines.md#tqllm" + } + ], + "tuning": [ + { + "id": "vram-tiers", + "title": "Pick the VRAM tier first", + "rule": "Choose how much of the model sits in VRAM first, then the model.", + "metrics": [ + { + "label": "Everything on the GPU, typical decode", + "before": null, + "after": "45-175", + "unit": "tok/s" + }, + { + "label": "A few expert layers in RAM, typical decode", + "before": null, + "after": "60-95", + "unit": "tok/s" + }, + { + "label": "Experts mostly in RAM, typical decode", + "before": null, + "after": "3-26", + "unit": "tok/s" + }, + { + "label": "Flint Q4_K_XL decode, 15 to 24 expert layers in RAM", + "before": 81.4, + "after": 64.2, + "unit": "tok/s" + }, + { + "label": "Flint Q4_K_XL prefill of 11.7k prompt, 15 to 24 layers", + "before": 2208, + "after": 1624, + "unit": "tok/s" + }, + { + "label": "Ornith-1.5-122B decode, 42 to 40 expert layers on CPU", + "before": 13.7, + "after": 16.3, + "unit": "tok/s" + } + ], + "chart": "tiers", + "evidence": "Flint Q4_K_XL, 2026-10-21/22: 81.4, 73.1, 64.2 tok/s at 15, 19, 24 expert layers in RAM.", + "caveat": "The expert cache moved the bottom tier up, not out.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#1-vram-tiers-pick-the-tier-first" + }, + { + "id": "expert-cache", + "title": "Cache hot experts on Flash-Next", + "rule": "For Flash-Next, keep all experts in RAM and cache the hot ones on the GPU.", + "metrics": [ + { + "label": "Orcarouter IQ4_XS streamed decode", + "before": "12-14", + "after": 19, + "unit": "tok/s" + }, + { + "label": "Heretic2 IQ4XS-NGQ4 streamed decode", + "before": "11-14", + "after": 18, + "unit": "tok/s" + }, + { + "label": "Abliterated s1.5 IQ4_XS streamed decode", + "before": "11.5-14.6", + "after": "19-21", + "unit": "tok/s" + }, + { + "label": "Swift 1.5 IQ3_XXS streamed decode", + "before": "13-17", + "after": "23-26", + "unit": "tok/s" + }, + { + "label": "Unpruned IQ3_S streamed decode", + "before": "12.5-15", + "after": "21-23", + "unit": "tok/s" + }, + { + "label": "Prefill on fresh loads, 16k and 48k", + "before": 250, + "after": "415-427", + "unit": "tok/s" + } + ], + "chart": "bars", + "evidence": "2026-10-22, Flash-Next only, five GGUFs at 262k: cache beat layer split on all; live on 12 entries.", + "caveat": "Untested for Ornith and Flint; not yet run through tests 1 and 2.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#2-expert-placement--ncmoe-against--cmoe-plus-the-expert-cache" + }, + { + "id": "kv-precision", + "title": "Use 8-bit KV when it buys room", + "rule": "Use 8-bit KV when it buys context, expert cache or batch; else f16.", + "metrics": [ + { + "label": "Mean KL, f16 KV against 8-bit KV plus cache", + "before": null, + "after": 0.0011, + "unit": "KL" + }, + { + "label": "Mean KL between two loads of the new config (noise floor)", + "before": null, + "after": 0.0002, + "unit": "KL" + }, + { + "label": "Top-1 agreement, old against new config", + "before": null, + "after": 98.2, + "unit": "%" + }, + { + "label": "Top-1 agreement between two loads (noise floor)", + "before": null, + "after": 99.6, + "unit": "%" + }, + { + "label": "Needles found in both configs", + "before": null, + "after": 8, + "unit": "of 8" + }, + { + "label": "vLLM window, FP8 KV against 4/2-bit KV", + "before": 150, + "after": 240, + "unit": "k tokens" + } + ], + "chart": "bars", + "evidence": "2026-10-22, Orcarouter GGUF: f16 KV against 8-bit KV plus cache gives mean KL 0.0011, top-1 98.2 %.", + "caveat": "KV type and cache could not be separated on 24 GB.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#3-kv-precision" + }, + { + "id": "batch-ubatch", + "title": "Raise ubatch until it stops loading", + "rule": "Take the largest -ub that still loads with everything else in place.", + "metrics": [ + { + "label": "Ornith prefill, 41 expert layers on CPU, -ub 2048", + "before": 116, + "after": 392, + "unit": "tok/s" + }, + { + "label": "Ornith decode, same change", + "before": 16.0, + "after": 16.1, + "unit": "tok/s" + }, + { + "label": "Flint prefill of 11.7k prompt at -ub 2048", + "before": null, + "after": 5357, + "unit": "tok/s" + } + ], + "chart": "slope", + "evidence": "Ornith, 41 expert layers on CPU, 2026-10-20: -ub 2048 lifted prefill from 116 to 392 tok/s.", + "caveat": "Too large fails: OrcaSAQ-2 needed -ub 256 at 22.7 of 24 GiB.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#4-batch-and-ubatch" + }, + { + "id": "context-margin", + "title": "Tell the agent a smaller context", + "rule": "Give the agent a context window smaller than the server's by max(10%, 16k).", + "metrics": [ + { + "label": "Compaction point on a 150k vLLM server", + "before": 71000, + "after": 89328, + "unit": "tokens" + }, + { + "label": "Share of window at compaction", + "before": 47, + "after": 60, + "unit": "%" + }, + { + "label": "Compactions in one 57k vLLM test", + "before": null, + "after": 55, + "unit": "times" + }, + { + "label": "Compactions with the Pi agent on the 57k setup", + "before": null, + "after": 8, + "unit": "times" + } + ], + "chart": "slope", + "evidence": "2026-10-04: a 240k vLLM case died at 213,761 prompt plus 32,000 reply against a 245,760 window.", + "caveat": "Runs before 2026-10-14, all of test 2, used the old window.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#5-context-margin-and-thinking-budget" + }, + { + "id": "thinking-budget", + "title": "Cap thinking inside the reply cap", + "rule": "Set the thinking budget to min(24000, reply cap minus 6000).", + "metrics": [ + { + "label": "Round 1 endpoints rescued, reasoning off", + "before": null, + "after": 6, + "unit": "of 9" + }, + { + "label": "Round 1 endpoints rescued, budget 4096", + "before": null, + "after": 8, + "unit": "of 9" + }, + { + "label": "Round 1 endpoints rescued, budget 16384", + "before": null, + "after": 4, + "unit": "of 9" + }, + { + "label": "Length of the one runaway think (OrcaSAQ-2)", + "before": null, + "after": 87000, + "unit": "characters" + } + ], + "chart": "bars", + "evidence": "2026-10-04: a cap of 23,961 under a flat 24,000 budget let one 87,000-character think eat the reply.", + "caveat": "A budget stops runaway thinking, not output loops.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#5-context-margin-and-thinking-budget" + }, + { + "id": "maker-sampling", + "title": "Serve the maker's sampling", + "rule": "Serve the maker's recommended sampling on the server; coding agents usually send none.", + "metrics": [ + { + "label": "Ternary Bonsai 2 test 1 score, with the makers' agent settings", + "before": 2, + "after": 5, + "unit": "of 5" + }, + { + "label": "Tiel Coder Q4_K_XL test 2 decode, makers' temperature 0.6", + "before": null, + "after": 78.6, + "unit": "tok/s" + }, + { + "label": "Tiel Coder Q4_K_XL test 2 duration", + "before": null, + "after": 27, + "unit": "minutes" + } + ], + "chart": "slope", + "evidence": "Adopted for every entry 2026-10-09 after loops ruined several runs; Bonsai 2 test 1 rose 2/5 to 5/5.", + "caveat": "Not a cure-all: Devstral Small 2, Qwen3.6-35B, GLM-4.7-Flash still failed.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#6-maker-sampling" + }, + { + "id": "speculative-decoding", + "title": "Draft only when the model fits", + "rule": "Use speculative decoding when the model is on the GPU; check answers stay unchanged.", + "metrics": [ + { + "label": "Qwen3.8-27B code, plain against llama.cpp MTP", + "before": 42, + "after": 93, + "unit": "tok/s" + }, + { + "label": "Qwen3.8-27B code, plain against ExLlamaV3 MTP", + "before": 42, + "after": 116, + "unit": "tok/s" + }, + { + "label": "Qwen3.8-27B code, plain against ExLlamaV3 DFlash2", + "before": 42, + "after": 158, + "unit": "tok/s" + }, + { + "label": "Qwen3.8-27B code, plain against TensorFold DFlash2", + "before": 42, + "after": 170, + "unit": "tok/s" + }, + { + "label": "Gemma 4 26B-A4B with MTP drafter, short code against 141k of history", + "before": 267, + "after": 123, + "unit": "tok/s" + }, + { + "label": "Ornith-1.5-122B experts in RAM, no draft against MTP with two drafts", + "before": 16.3, + "after": 16.7, + "unit": "tok/s" + }, + { + "label": "Ornith-1.5-122B, no draft against a DFlash drafter", + "before": 16.3, + "after": 11.5, + "unit": "tok/s" + }, + { + "label": "Maya-M IQ2_S, no draft against DFlash2 draft", + "before": 15.35, + "after": 10.7, + "unit": "tok/s" + } + ], + "chart": "bars", + "evidence": "2026-10-13, Qwen3.8-27B code, one card: 42 tok/s plain, 170 with TensorFold DFlash2; Ornith in RAM gained nothing.", + "caveat": "At temperature 0 drafting changed 7 of 15 DFlash2 replies; TensorFold none.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#7-speculative-decoding-when-it-pays" + }, + { + "id": "warm-cache", + "title": "Warm up and measure twice", + "rule": "Warm up before timing; run each setting twice in opposite orders; never rank on one run.", + "metrics": [ + { + "label": "Strata 0.1.42 prompt reading, cold against warm", + "before": 44, + "after": 1269, + "unit": "tok/s" + }, + { + "label": "Strata 0.1.42 decode, cold against warm round", + "before": 13.1, + "after": 23.5, + "unit": "tok/s" + }, + { + "label": "Unpruned Flash-Next GGUF, raw benchmark against streamed", + "before": 40, + "after": "21-23", + "unit": "tok/s" + }, + { + "label": "Prefill, quality-script runs (not reproduced) → fresh load", + "before": "85-100", + "after": 415, + "unit": "tok/s" + } + ], + "chart": "slope", + "evidence": "Strata 0.1.42, 2026-10-11: the same prompt read at 44 tok/s cold and 1,269 warm.", + "caveat": "Differences smaller than the second-round speedup were treated as noise.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#8-cold-against-warm-page-cache-and-measuring-twice" }, { - "model": "maya-glm-5.3-flash", - "engine": "Maya 1.0.4", - "variant": "2 rounds, 3.5 bpw Maya-S", - "decode": 14.6, - "quote": null, - "prefill": null, - "vram": null, - "error": "" + "id": "pcie-share", + "title": "Measure the PCIe share setting", + "rule": "Measure an engine's GPU-CPU transfer share on your own card; release defaults can be wrong.", + "metrics": [ + { + "label": "Strata 0.1.42 long-answer decode, default against old share", + "before": 48, + "after": 95, + "unit": "tok/s" + }, + { + "label": "Strata 0.1.42 big-prompt reading", + "before": 1177, + "after": 1983, + "unit": "tok/s" + }, + { + "label": "Test 1 duration", + "before": 94, + "after": 51, + "unit": "minutes" + }, + { + "label": "Test 2 duration", + "before": 98, + "after": 61, + "unit": "minutes" + }, + { + "label": "Test 2 tests passed", + "before": 72, + "after": 77, + "unit": "tests" + }, + { + "label": "Maya-M decode, share 0.36 against 0.25 (not applied)", + "before": 16.0, + "after": 17.9, + "unit": "tok/s" + } + ], + "chart": "slope", + "evidence": "Strata 0.1.42 on Flash-Next, 2026-10-11: older PCIe share doubled long-answer decode, 48 to 95 tok/s.", + "caveat": "Maya-M's +12 % candidate was measured 2026-10-17 and not applied.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#9-pcie-share-and-other-engine-internal-knobs" }, { - "model": "maya1111-glm-5.3-flash", - "engine": "Maya 1.1.11", - "variant": "2 rounds, same pack", - "decode": 17.0, - "quote": null, - "prefill": null, - "vram": null, - "error": "" + "id": "engine-choice", + "title": "Choose the engine for the model", + "rule": "Choose the engine for the model class, then tune; short-request speed is not agent readiness.", + "metrics": [ + { + "label": "Flash-Next, experts in RAM: llama.cpp 0.4 (2026-10-05)", + "before": null, + "after": 2.2, + "unit": "tok/s" + }, + { + "label": "Flash-Next, experts in RAM: llama.cpp 0.6 (2026-10-05)", + "before": null, + "after": 8.9, + "unit": "tok/s" + }, + { + "label": "Flash-Next: Strata (2026-10-04)", + "before": null, + "after": "30-45", + "unit": "tok/s" + }, + { + "label": "Flash-Next: llama.cpp 0.7.0 with cache, streamed (2026-10-22)", + "before": null, + "after": "19-26", + "unit": "tok/s" + }, + { + "label": "Flash-Next Gyro-S on Strata 0.2.2, code (2026-10-22)", + "before": null, + "after": 144, + "unit": "tok/s" + }, + { + "label": "Multi-session checks passed: tqLLM (best) against stock SGLang", + "before": 4, + "after": 11, + "unit": "of 11" + } + ], + "chart": "timeline", + "evidence": "Flash-Next with experts in RAM: 2.2 tok/s on llama.cpp 0.4, 30-45 on Strata 0.1 (2026-10-04); Gyro-S on Strata 0.2.2 reached 144 on code (2026-10-22).", + "caveat": "Quants differ across rows; Strata's 144 is Gyro-S on code.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#10-engine-choice" }, { - "model": "exl3-qwen3.8-27b-dflash2", - "engine": "exllamav3 1.6.1", - "variant": "2 rounds, DFlash2, thinking off", - "decode": 138.6, - "quote": null, - "prefill": null, - "vram": null, - "error": "" + "id": "quality-checks", + "title": "Check a setup before trusting speed", + "rule": "Run quality and multi-session checks on every new engine or settings change before ranking speed.", + "metrics": [ + { + "label": "Code decode on the same GGUF, llama.cpp → tqLLM", + "before": 40.3, + "after": 101.7, + "unit": "tok/s" + }, + { + "label": "Multi-session checks passed, tqLLM (best) → llama.cpp live", + "before": 4, + "after": 11, + "unit": "of 11" + }, + { + "label": "Multi-session checks passed, Carbon-X at one slot", + "before": null, + "after": 10, + "unit": "of 11" + }, + { + "label": "Old runs regraded, 2026-10-10", + "before": null, + "after": 125, + "unit": "runs" + }, + { + "label": "Verdicts changed by the regrade", + "before": null, + "after": 4, + "unit": "verdicts" + } + ], + "chart": "bars", + "evidence": "2026-10-22: tqLLM decoded about 2.5 times faster than llama.cpp yet passed at best 4 of 11 multi-session checks.", + "caveat": "A speed result without the probe next to it is a lead, not a pick.", + "doc": "https://github.com/alexderz/agentic-model-testing/blob/main/docs/model-research/tuning.md#11-the-quality-checks-we-use" + } + ], + "quality_checks": [ + { + "id": "kl-top1", + "name": "KL and top-1 agreement", + "what": "Greedy logprobs old against new config, same GGUF, cuts of 2k-100k tokens", + "threshold": "Compare with the noise floor: KL 0.0002, top-1 99.6 %", + "example": "Expert cache, 2026-10-22: KL 0.0011, top-1 98.2 %" }, { - "model": "exl3-qwen3.8-27b-dflash2-162", - "engine": "exllamav3 1.6.2", - "variant": "2 rounds, DFlash2, thinking off", - "decode": 140.1, - "quote": null, - "prefill": null, - "vram": null, - "error": "" + "id": "needle", + "name": "Needle in a haystack", + "what": "Planted code in filler text, 16k to 250k tokens, room left for thinking", + "threshold": null, + "example": "Stock SGLang needles at 100k, 150k, 190k on two of three builds" }, { - "model": "strata020-gyro-s-mtp-test", - "engine": "Strata 0.2.0-beta", - "variant": "Gyro-S + MTP, code prompt", - "decode": 118.6, - "quote": null, - "prefill": 2550.9, - "vram": 23862, - "error": "" + "id": "digit-copy", + "name": "Digit-copy probe", + "what": "Copies digit strings in long prompts and finds two needles", + "threshold": "12 of 12 strings and both needles", + "example": "Maya 1.0.4 split one string; 1.1.11 copied 12 of 12" }, { - "model": "strata44-flash-next-pcie", - "engine": "Strata 0.2.0-beta", - "variant": "IQ3_S + MTP, code prompt", - "decode": 35.4, - "quote": null, - "prefill": 2645.4, - "vram": 23868, - "error": "" + "id": "tool-call", + "name": "Tool-call probe", + "what": "One function call with arguments, parsed", + "threshold": null, + "example": "Passed on every engine until tqLLM's first failure" }, { - "model": "strata44-flash-next-pcie", - "engine": "Strata 0.1.45", - "variant": "IQ3_S + MTP, code prompt", - "decode": 29.4, - "quote": null, - "prefill": 2597.8, - "vram": 23874, - "error": "" + "id": "multi-session", + "name": "Multi-session probe", + "what": "11 checks: parallel answers, two decoding together, tool calls, long plus short, five concurrent", + "threshold": "11 of 11 (10 of 11 at one slot, by design)", + "example": "llama.cpp live 11 of 11; tqLLM 3, 0 and 4 of 11" }, { - "model": "strata020-gyro-s-mtp-test", - "engine": "Strata 0.2.2", - "variant": "Gyro-S + MTP, code prompt", - "decode": 144.1, - "quote": null, - "prefill": 2545.7, - "vram": 23862, - "error": "" + "id": "agent-tests", + "name": "Real agent tests", + "what": "Tests 1 to 8; the only check that sees what an agent does", + "threshold": null, + "example": "Gyro-S + MTP test 2: full pass in 24 minutes" }, { - "model": "strata020-gyro-s-mtp-test", - "engine": "Strata 0.2.0-beta", - "variant": "Gyro-S + MTP, code prompt, same-day control", - "decode": 121.6, - "quote": null, - "prefill": 2539.8, - "vram": 23862, - "error": "" + "id": "grader-audits", + "name": "Grader audits", + "what": "Regrade old runs when a grader changes", + "threshold": null, + "example": "125 runs regraded 2026-10-10; four verdicts changed" } ], "oneshots": { @@ -9146,6 +11702,78 @@ "note": "no SVG: length stop at 32,000 tokens, all thinking (five identical replies)", "frames": [], "live": null + }, + { + "case": "mx22-flashnext-heretic2-t5-oneshot-orrery-r1", + "entry": "lcpp6-flashnext-heretic2", + "test": 5, + "finish": "stop", + "tokens": 33547.0, + "wall_s": 1513.4, + "tok_s": 22.3, + "points": { + "A": 0.0, + "B": 0.0, + "C": 0.0, + "D": 0.0, + "E": 0.0 + }, + "total": 0.0, + "note": "Final model-supplied standalone SVG is malformed (mismatched tag, line 77). The harness also greedily concatenated two SVG blocks/prose: that packaging error was separated by extracting the final original codeblock verbatim, which still fails XML parsing. Initial valid background-only fragment is not substituted or graded.", + "frames": [ + "frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-1500.webp", + "frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-5000.webp", + "frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-10000.webp" + ], + "live": null + }, + { + "case": "t5nm1-lcpp6-ornith-122b-q4kxl", + "entry": "lcpp6-ornith-122b-q4kxl", + "test": 5, + "finish": "stop", + "tokens": 29646.0, + "wall_s": 1940.1, + "tok_s": 15.3, + "points": { + "A": 1.0, + "B": 1.0, + "C": 0.0, + "D": 1.0, + "E": 0.0 + }, + "total": 3.0, + "note": "Parses and draws, but planets are left at the central origin instead of translated to arm ends. Only gears visibly move; outer markers stay fixed across sealed frames. Nominal period parameters are not functioning planetary orbits. Toothed wheels turn without credible meshing; no visible planetary day/night.", + "frames": [ + "frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-1500.webp", + "frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-5000.webp", + "frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-10000.webp" + ], + "live": "live/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/orrery.svg" + }, + { + "case": "t5nm1-lcpp-flint-40b-q3kxl", + "entry": "lcpp-flint-40b-q3kxl", + "test": 5, + "finish": "stop", + "tokens": 30402.0, + "wall_s": 299.0, + "tok_s": 101.8, + "points": { + "A": 1.0, + "B": 2.0, + "C": 0.0, + "D": 0.0, + "E": 1.0 + }, + "total": 4.0, + "note": "Renders with detached teeth and shadow halos; motion present; orbital periods 80/38/20/40/160/300s incorrectly order inner planets; no credible meshing pairs; offset black shadows rotate incorrectly. Independent original-source/frame review.", + "frames": [ + "frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-1500.webp", + "frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-5000.webp", + "frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-10000.webp" + ], + "live": "live/orrery/t5nm1-lcpp-flint-40b-q3kxl/orrery.svg" } ], "pagoda": [ @@ -10256,6 +12884,78 @@ "frames/pagoda/t6unc1-qwen3.8-27b-uncensored-vllm/frame-10000.webp" ], "live": "live/pagoda/t6unc1-qwen3.8-27b-uncensored-vllm/page.html" + }, + { + "case": "mx22-flashnext-heretic2-t6-oneshot-pagoda-r1", + "entry": "lcpp6-flashnext-heretic2", + "test": 6, + "finish": "stop", + "tokens": 29545.0, + "wall_s": 1359.9, + "tok_s": 21.8, + "points": { + "A": 0.0, + "B": 0.0, + "C": 0.0, + "D": 0.0, + "E": 0.0 + }, + "total": 0.0, + "note": "Original animation throws ReferenceError: clock is not defined before scene rendering; all sealed frames contain only the clear-color background. Offline three.js dependency packaging succeeds. Blank-page rule scores every dimension zero; no diagnostic model-code fix was scored.", + "frames": [ + "frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-1500.webp", + "frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-5000.webp", + "frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-10000.webp" + ], + "live": null + }, + { + "case": "t6nm1-lcpp6-ornith-122b-q4kxl", + "entry": "lcpp6-ornith-122b-q4kxl", + "test": 6, + "finish": "stop", + "tokens": 6653.0, + "wall_s": 523.0, + "tok_s": 15.4, + "points": { + "A": 1.0, + "B": 2.0, + "C": 1.0, + "D": 2.0, + "E": 2.0 + }, + "total": 8.0, + "note": "Draws consistently from boxes; five tiers/upturned eaves but full spire is cropped in every frame. Platform, cherry trees, pond and torii visible. Falling petals, shadows and frame-based camera orbit present.", + "frames": [ + "frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-1500.webp", + "frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-5000.webp", + "frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-10000.webp" + ], + "live": "live/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/page.html" + }, + { + "case": "t6nm1-lcpp-flint-40b-q3kxl", + "entry": "lcpp-flint-40b-q3kxl", + "test": 6, + "finish": "stop", + "tokens": 15958.0, + "wall_s": 145.2, + "tok_s": 110.1, + "points": { + "A": 1.0, + "B": 2.0, + "C": 1.0, + "D": 2.0, + "E": 2.0 + }, + "total": 8.0, + "note": "Draws consistently from boxes; five tiers but complete spire cropped in all three frames; platform pond cherry tree and partly occluded torii visible; petals shadows and frame-based camera orbit present. Offline CDN rewrite is packaging only; no model fix.", + "frames": [ + "frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-1500.webp", + "frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-5000.webp", + "frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-10000.webp" + ], + "live": "live/pagoda/t6nm1-lcpp-flint-40b-q3kxl/page.html" } ], "agent-orrery": [ @@ -10768,12 +13468,12 @@ ] }, "counts": { - "models": 93, - "runs": 206, - "test3": 51, - "oneshots": 100, - "frames": 225, - "pass": 133, - "live": 74 + "models": 95, + "runs": 222, + "test3": 53, + "oneshots": 106, + "frames": 243, + "pass": 148, + "live": 78 } } diff --git a/site/data/curated.json b/site/data/curated.json index 7a384fa..dd1fb0a 100644 --- a/site/data/curated.json +++ b/site/data/curated.json @@ -62,15 +62,15 @@ "findings": [ { "date": "2026-10-22", - "title": "An expert cache doubles Flash-Next on one 3090", + "title": "An expert cache makes Flash-Next 1.5x faster on one 3090", "kind": "tuning", "summary": "Qwen3.8 Flash-Next (125B, 6B active) does not fit in 24 GB, so llama.cpp keeps experts in system RAM. Keeping all of them there (-cmoe) and giving the GPU a cache of hot experts (--moe-cache-mib), with q8 KV to make room, beat the old split of 40 layers on CPU on every GGUF tried, at 262k context.", "settings": "-cmoe --moe-cache-mib 12800 -ctk q8_0 -ctv q8_0 -b 2048 -ub 1024 -t 16", "metrics": [ { - "label": "Decode, orca IQ4_XS", + "label": "Decode, orca IQ4_XS (streamed, live config)", "before": 13, - "after": 24, + "after": 19, "unit": "tok/s" }, { diff --git a/site/data/deep.json b/site/data/deep.json new file mode 100644 index 0000000..f7dd966 --- /dev/null +++ b/site/data/deep.json @@ -0,0 +1,511 @@ +{ + "schema": "deep/1", + "engines": [ + { + "id": "llama-cpp", + "name": "llama.cpp", + "kind": "GGUF server; dense models and MoE with experts in RAM", + "versions": [ + {"version": "0.4.1-dev (commit 4260903)", "date": "2026-09-23 to 2026-10-18", "note": "Bundled build used by the native entries; Flash-Next with experts in RAM: 2.2 tok/s on the probe; long agent runs on it ran out of time."}, + {"version": "0.6.0 (official image, build 11429)", "date": "2026-10-05", "note": "2.5-4x faster than 0.4 on Flash-Next with experts in RAM; MTP and n-gram drafting fixes."}, + {"version": "0.7.0", "date": "2026-10-18", "note": "Released 2026-10-15. Adds a GPU cache for experts kept in RAM (--moe-cache-mib); in the router since 2026-10-18."}, + {"version": "Newer official builds", "date": "2026-10-21", "note": "Flint needs b11803 or later; Carbon-X needs b11742 or later; trials ran by hand, not in the router."} + ], + "good_at": "Any GGUF, including MoE models with experts in system RAM", + "watch_out": "Check the build supports the model's architecture before reading any result", + "multi_session": {"ok": true, "note": "11 of 11 probe checks on Flash-Next entries"}, + "speeds": [ + {"model": "Tiel Coder 35B IQ4_XS", "config": "0.7.0, all on GPU", "decode": 151, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-18"}, + {"model": "Flint 40B-A2.5B Q3_K_XL", "config": "newer build, all on GPU, q8 KV, short prompt", "decode": 129, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flint 40B-A2.5B Q3_K_XL", "config": "newer build, all on GPU, q8 KV, 11.7k depth", "decode": 106, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flint Q4_K_XL", "config": "10 expert layers in RAM", "decode": 95, "prefill": null, "unit": "tok/s", "context": "150k", "date": "2026-10"}, + {"model": "Flint Q4_K_XL", "config": "15 expert layers in RAM; prefill of an 11.7k prompt", "decode": 81.4, "prefill": 2208, "unit": "tok/s", "context": null, "date": "2026-10-22"}, + {"model": "Flint Q4_K_XL", "config": "19 expert layers in RAM", "decode": 73.1, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-22"}, + {"model": "Flint Q4_K_XL", "config": "24 expert layers in RAM; prefill of an 11.7k prompt", "decode": 64.2, "prefill": 1624, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Ornith-1.5-122B UD-Q4_K_XL", "config": "40 expert layers on CPU", "decode": 16.3, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-20"}, + {"model": "Carbon-X 14.8B Q8_0", "config": "all on GPU", "decode": 52, "prefill": null, "unit": "tok/s", "context": "40k", "date": "2026-10-22"}, + {"model": "Qwen3.8-27B", "config": "MTP drafting, all on GPU, code", "decode": 93, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-13"}, + {"model": "Qwen3.8-27B", "config": "DFlash2 drafter (GGUF), code; 47 % of drafts accepted", "decode": 95, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-13"}, + {"model": "Qwen3.8-27B", "config": "DFlash2 drafter (GGUF), prose", "decode": 43, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-13"}, + {"model": "Qwen3.8-27B", "config": "DFlash2 drafter (GGUF), structured output", "decode": 108, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-13"}, + {"model": "Flash-Next Orcarouter IQ4_XS", "config": "0.7.0, -cmoe, cache 12800 MiB, q8 KV, streamed (about)", "decode": 19, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flash-Next Heretic2 IQ4XS-NGQ4", "config": "0.7.0, -cmoe, cache 12800 MiB, q8 KV, streamed (about)", "decode": 18, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flash-Next Abliterated s1.5 IQ4_XS", "config": "0.7.0, -cmoe, cache 13824 MiB, q8 KV, streamed", "decode": "19-21", "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flash-Next Swift 1.5 IQ3_XXS (pruned)", "config": "0.7.0, -cmoe, cache 9216 MiB, q8 KV, streamed", "decode": "23-26", "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flash-Next unpruned IQ3_S", "config": "0.7.0, -cmoe, cache 9216 MiB, q8 KV, streamed", "decode": "21-23", "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flash-Next GGUFs (fresh loads)", "config": "0.7.0 expert cache; prefill about 250 before", "decode": null, "prefill": "415-427", "unit": "tok/s", "context": "16k and 48k", "date": "2026-10-22"}, + {"model": "Flash-Next unpruned IQ3_S", "config": "0.7.0, raw short outputs, thinking off, code; prose 41.3, structured 41.0", "decode": 40.3, "prefill": null, "unit": "tok/s", "context": "32k", "date": "2026-10-22"}, + {"model": "Qwen3.8-Flash-Next", "config": "0.4.1-dev, experts in RAM, short probe", "decode": 2.2, "prefill": 136, "unit": "tok/s", "context": "49k", "date": "2026-10-05"}, + {"model": "Qwen3.8-Flash-Next", "config": "0.6, experts in RAM, short probe", "decode": 8.9, "prefill": 569, "unit": "tok/s", "context": "49k", "date": "2026-10-05"}, + {"model": "Orcarouter Flash-Next Uncensored", "config": "0.4.1-dev, experts in RAM, short probe", "decode": 5.3, "prefill": 295, "unit": "tok/s", "context": "49k", "date": "2026-10-05"}, + {"model": "Orcarouter Flash-Next Uncensored", "config": "0.6, experts in RAM, short probe", "decode": 13.1, "prefill": 486, "unit": "tok/s", "context": "49k", "date": "2026-10-05"}, + {"model": "Qwen3.8-27B Cyber", "config": "0.6, all on GPU, no drafting", "decode": 45.1, "prefill": null, "unit": "tok/s", "context": "49k", "date": "2026-10-05"}, + {"model": "Qwen3.8-27B Cyber", "config": "0.6, all on GPU, with MTP", "decode": 82.2, "prefill": null, "unit": "tok/s", "context": "49k", "date": "2026-10-05"}, + {"model": "OrcaSAQ-2-Cyber-27B", "config": "0.6, all on GPU, no drafting", "decode": 83.2, "prefill": null, "unit": "tok/s", "context": "49k", "date": "2026-10-05"}, + {"model": "OrcaSAQ-2-Cyber-27B", "config": "0.6, n-gram drafts, copy-heavy output", "decode": 412, "prefill": null, "unit": "tok/s", "context": "49k", "date": "2026-10-05"}, + {"model": "Cyber-Tiel-Coder-35B", "config": "0.6, 20 expert layers in RAM, short probe", "decode": 50.5, "prefill": null, "unit": "tok/s", "context": "49k", "date": "2026-10-05"} + ], + "bugs": [ + {"text": "0.4.1-dev ran Flash-Next with experts in RAM 2.5-4x slower than 0.6", "status": "fixed", "date": "2026-10-05"}, + {"text": "0.6: automatic -fit aborts when -ngl is set by hand", "status": "worked around", "date": null}, + {"text": "0.6: offloaded models stall on the first request without -lm none", "status": "worked around", "date": "2026-09-30"}, + {"text": "0.7.0: OrcaSAQ-2 with DFlash2 drafts failed every request on 580 MiB buffer", "status": "worked around", "date": "2026-10-18"}, + {"text": "Qwen3.8-27B Cyber on 0.7.0: empty 32,000-token replies once, unrepeated", "status": "open", "date": null}, + {"text": "Lazy tensor reads slow the first pass over unseen text when cache is cold", "status": "open", "date": null}, + {"text": "Newer-architecture GGUFs load on older builds, then fail or tokenize wrongly", "status": "worked around", "date": "2026-10-21"} + ], + "doc": "docs/model-research/engines.md#llamacpp" + }, + { + "id": "vllm", + "name": "vLLM (HyperQwen)", + "kind": "dense model server with prefix caching", + "versions": [ + {"version": "HyperQwen 0.30", "date": null, "note": "In use; 2-4x faster than llama.cpp over whole agent runs on Qwen3.8-27B."}, + {"version": "0.31", "date": "2026-10-14", "note": "Same speed rounds, slightly slower deep (79.5 against 84.5 tok/s at 141k); not adopted."}, + {"version": "0.32", "date": null, "note": "Image built and staged; not in use."} + ], + "good_at": "Dense models on the GPU; long agent runs with prefix caching", + "watch_out": "Needs per-request thinking budget; images only with VISION=1", + "multi_session": {"ok": true, "note": "Main agent plus sub-agents ran cleanly, tests 3 and 8"}, + "speeds": [ + {"model": "Qwen3.8-27B", "config": "MTP drafts, FP8 KV, average over a run (about)", "decode": 70, "prefill": null, "unit": "tok/s", "context": "150k", "date": null}, + {"model": "Qwen3.8-27B", "config": "MTP drafts, FP8 KV, short code", "decode": 127, "prefill": null, "unit": "tok/s", "context": "150k", "date": null}, + {"model": "Qwen3.8-27B", "config": "MTP drafts, FP8 KV, 141k of history", "decode": 85, "prefill": null, "unit": "tok/s", "context": "141k history", "date": null}, + {"model": "Qwen3.8-27B", "config": "DFlash2 drafts, 4/2-bit KV, about half the speed under load", "decode": null, "prefill": null, "unit": "tok/s", "context": "240k", "date": null}, + {"model": "Qwen3.8-27B", "config": "DFlash2 drafts (about)", "decode": 97, "prefill": null, "unit": "tok/s", "context": "64k", "date": "2026-10-06"}, + {"model": "Qwen3.8-27B", "config": "DFlash2 15-token drafts (about)", "decode": 95, "prefill": null, "unit": "tok/s", "context": "57k", "date": "2026-10-06"} + ], + "bugs": [ + {"text": "With thinking on, Qwen3.8-27B finished no one-shot pagoda in seven tries", "status": "open", "date": "2026-10-14"}, + {"text": "Gemma 4 26B's server passed the grader; a real client could not connect", "status": "open", "date": "2026-10-10"}, + {"text": "DFlash2 15-token drafts at 57k: one test compacted 55 times", "status": "worked around", "date": "2026-10-06"}, + {"text": "DFlash2 at 64k: agent compacts constantly; one test overflowed the context", "status": "worked around", "date": "2026-10-06"} + ], + "doc": "docs/model-research/engines.md#vllm" + }, + { + "id": "sglang", + "name": "SGLang (EXL3)", + "kind": "MoE server with GPU expert cache, EXL3 weights", + "versions": [ + {"version": "Original fork build", "date": "2026-10-01 to 2026-10-04", "note": "200k, fp8 KV; built the best test-2 server of all runs: 12 tools, 155 tests, 124 minutes."}, + {"version": "Newer fork build (phase D)", "date": "2026-10-11", "note": "11 of 11 multi-session checks; test 2 cut off and never graded; dropped 2026-10-21."}, + {"version": "Three stock builds", "date": "2026-10-21", "note": "204,800 context, two slots, EXL3 3.05 bpw, 20.8-21.4 GB, load 252-262 s; needles at 100k, 150k, 190k passed on two of three builds (8-bit KV build not run)."} + ], + "good_at": "Most capable Flash-Next setup tested: images, 200k window", + "watch_out": "Slower than the 27B stacks; no agent test rerun on stock builds", + "multi_session": {"ok": true, "note": "Stock builds: 11 of 11 checks, two slots"}, + "speeds": [ + {"model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", "config": "original fork build, fp8 KV, average over the test-2 run", "decode": 36, "prefill": null, "unit": "tok/s", "context": "200k", "date": "2026-10-04"}, + {"model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", "config": "original fork build, test 1", "decode": 44.3, "prefill": null, "unit": "tok/s", "context": "200k", "date": "2026-10-04"}, + {"model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", "config": "newer fork build, test 1, earlier settings", "decode": 25.3, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-11"}, + {"model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", "config": "stock build 1, two slots, prefill at 190k", "decode": 24.1, "prefill": 2600, "unit": "tok/s", "context": "204,800", "date": "2026-10-21"}, + {"model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", "config": "stock build 2, two slots, prefill at 190k", "decode": 24.6, "prefill": 2600, "unit": "tok/s", "context": "204,800", "date": "2026-10-21"}, + {"model": "Qwen3.8-Flash-Next (EXL3 3.05 bpw)", "config": "stock build 3, two slots, prefill at 190k", "decode": 25.1, "prefill": 2600, "unit": "tok/s", "context": "204,800", "date": "2026-10-21"} + ], + "bugs": [ + {"text": "Original build served one session at a time and crashed under several", "status": "fixed", "date": "2026-10-03"}, + {"text": "A test-3 run on the original build needed Strata instead", "status": "worked around", "date": null} + ], + "doc": "docs/model-research/engines.md#sglang" + }, + { + "id": "strata", + "name": "Strata", + "kind": "MoE server with hot-expert VRAM cache and built-in MTP", + "versions": [ + {"version": "0.1.39 / 0.1.40.3", "date": "2026-10-04 to 2026-10-09", "note": "30-45 tok/s decode, 21-30 over a whole run; test 2 full passes with Flash-Next and Swift 1.5."}, + {"version": "0.1.42", "date": "2026-10-10 to 2026-10-12", "note": "Release defaults slower; the old PCIe share doubled long-answer decode."}, + {"version": "0.1.44.1", "date": "2026-10-15", "note": "Load 60 s against 214 s; decode 44.9 against 18.3 on 0.1.42 (mean of two rounds: cold 13.1, warm 23.5)."}, + {"version": "0.1.45", "date": "2026-10-20", "note": "IQ3_S with MTP, code prompt: 29.4 tok/s."}, + {"version": "0.2.0-beta", "date": "2026-10-20", "note": "Gyro-S with MTP: test 2 full pass in 24 minutes, 89.8 tok/s over the run; agent pagoda 9 of 10."}, + {"version": "0.2.2", "date": "2026-10-22", "note": "Gyro-S: code 121.6 to 144.1 tok/s on the same-day control, load 23 s; no agent run yet."} + ], + "good_at": "Fastest Flash-Next decode measured; made 1-bit-class Gyro-S usable", + "watch_out": "0.2 is beta; warm up before timing; set reasoning budget", + "multi_session": {"ok": true, "note": "Works with parallel 2 and --batch-mtp; Gyro build unprobed"}, + "speeds": [ + {"model": "Qwen3.8-Flash-Next (GSQ-RCO quants)", "config": "0.1.39 / 0.1.40.3, decode; 21-30 over a whole run", "decode": "30-45", "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-09"}, + {"model": "Qwen3.8-Flash-Next", "config": "0.1.42 release-default PCIe share, long answers", "decode": 48, "prefill": 1177, "unit": "tok/s", "context": null, "date": "2026-10-11"}, + {"model": "Qwen3.8-Flash-Next", "config": "0.1.42 old PCIe share, long answers", "decode": 95, "prefill": 1983, "unit": "tok/s", "context": null, "date": "2026-10-11"}, + {"model": "Qwen3.8-Flash-Next", "config": "0.1.42, first round, cold", "decode": 13.1, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-15"}, + {"model": "Qwen3.8-Flash-Next", "config": "0.1.42, warm round", "decode": 23.5, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-15"}, + {"model": "Qwen3.8-Flash-Next", "config": "0.1.44.1", "decode": 44.9, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-15"}, + {"model": "Flash-Next IQ3_S", "config": "0.1.45, MTP, code prompt", "decode": 29.4, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-20"}, + {"model": "Flash-Next IQ3_S", "config": "0.2.0-beta, MTP, code prompt", "decode": 35.4, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-20"}, + {"model": "Flash-Next Gyro-S (TQ1_0)", "config": "0.2.0-beta, MTP draft, code, text only", "decode": 118.6, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-20"}, + {"model": "Flash-Next Gyro-S (TQ1_0)", "config": "0.2.0-beta, MTP, average over test-2 run", "decode": 89.8, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-20"}, + {"model": "Flash-Next Gyro-S (TQ1_0)", "config": "0.2.2, MTP, code", "decode": 144.1, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-22"}, + {"model": "Flash-Next Gyro-S (TQ1_0)", "config": "0.2.2, MTP, prose", "decode": 123.4, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-22"}, + {"model": "Flash-Next Gyro-S (TQ1_0)", "config": "0.2.2, MTP, JSON", "decode": 157.7, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-22"} + ], + "bugs": [ + {"text": "0.1.42 release-default PCIe share halved decode", "status": "worked around", "date": "2026-10-11"}, + {"text": "0.1.42 test 2T first start failed: engine exits locking about 40 GB host RAM", "status": "open", "date": null}, + {"text": "Gyro no-MTP arms fail to start; native pack requires speculative decoding", "status": "open", "date": null}, + {"text": "1.6-bit coder looped until reasoning_budget_tokens was set", "status": "worked around", "date": null}, + {"text": "DNS-rebinding guard returns 403 to any host name not in allowed_hosts", "status": "worked around", "date": null} + ], + "doc": "docs/model-research/engines.md#strata" + }, + { + "id": "maya", + "name": "Maya", + "kind": "tiered-expert MoE server (VRAM, RAM, SSD)", + "versions": [ + {"version": "1.0.4 with three patches", "date": "2026-10-08", "note": "About 4 tok/s out of the box; 17-19 with patches once weights were on local disk. Test 1 pass (44 tests), test 2 pass (7 tools, 66 tests, 322 minutes)."}, + {"version": "1.1.11", "date": "2026-10-16", "note": "Patches upstream; 17.0 against 14.6 tok/s, first token 18 s against 92 s, 12 of 12 digit strings."}, + {"version": "1.1.13", "date": "2026-10-17", "note": "Maya-S and Maya-M IQ2_S: 12 of 12 digits, 2 of 2 needles; test 3 rubric 12 of 12 on both."} + ], + "good_at": "Runs a 321B MoE on one card and passed both tests", + "watch_out": "Slowest model to pass both tests; first load reads 100 GB", + "multi_session": {"ok": null, "note": "Not measured; treated as one request at a time"}, + "speeds": [ + {"model": "GLM-5.3-Flash Maya-S IQ2_XXS", "config": "1.0.4 out of the box", "decode": 4, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-08"}, + {"model": "GLM-5.3-Flash Maya-S IQ2_XXS", "config": "1.0.4 with three patches, weights on local disk", "decode": "17-19", "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-08"}, + {"model": "GLM-5.3-Flash Maya-S IQ2_XXS", "config": "1.1.11, two probe rounds (1.0.x: 14.6 on the same 3.5 bpw pack)", "decode": 17.0, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-16"}, + {"model": "GLM-5.3-Flash Maya-S / Maya-M", "config": "1.1.13, test 3 runs, both flavors", "decode": "5.4-6.9", "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-17"}, + {"model": "GLM-5.3-Flash Maya-M IQ2_S", "config": "default PCIe share 0.36 (0.25 gave 17.9)", "decode": 16.0, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-17"}, + {"model": "GLM-5.3-Flash Maya-M IQ2_S", "config": "PCIe share 0.25 (candidate, not applied)", "decode": 17.9, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-17"}, + {"model": "GLM-5.3-Flash Maya-M IQ2_S", "config": "no DFlash2 draft head", "decode": 15.35, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-17"}, + {"model": "GLM-5.3-Flash Maya-M IQ2_S", "config": "with DFlash2 draft head, 91 % of drafts accepted", "decode": 10.7, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-17"} + ], + "bugs": [ + {"text": "1.0.4 decoded about 4 tok/s; every VRAM eviction re-read from the network", "status": "fixed", "date": "2026-10-08"}, + {"text": "On 1.0.x, GLM split multi-digit tool-argument numbers with spaces", "status": "fixed", "date": "2026-10-16"}, + {"text": "DFlash2 draft on Maya-M slowed decode to 10.7 from 15.35 tok/s", "status": "worked around", "date": "2026-10-17"}, + {"text": "1-bit Huihui variant failed both tests with repetition", "status": "open", "date": null} + ], + "doc": "docs/model-research/engines.md#maya" + }, + { + "id": "exllamav3", + "name": "ExLlamaV3 (TabbyAPI)", + "kind": "dense model server, EXL3 weights", + "versions": [ + {"version": "1.6.1", "date": "2026-10-13", "note": "Runs Qwen3.8-27B from EXL3 4.00 bpw, unpatched, with its MTP head or the DFlash2 drafter."}, + {"version": "1.6.2", "date": "2026-10-16", "note": "140.1 against 138.6 tok/s on 1.6.1 with DFlash2, within noise."}, + {"version": "1.6.3", "date": null, "note": "Image built and staged; not in use."} + ], + "good_at": "Larger window than vLLM for the 27B; DFlash2 without a patch", + "watch_out": "Slower than vLLM with MTP; drafting alters temperature-0 replies", + "multi_session": {"ok": null, "note": "Not measured"}, + "speeds": [ + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "DFlash2 drafter, code; 98k window fits with 8-bit cache", "decode": 158, "prefill": null, "unit": "tok/s", "context": "16k", "date": "2026-10-13"}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "DFlash2 drafter, prose", "decode": 73, "prefill": null, "unit": "tok/s", "context": "16k", "date": "2026-10-13"}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "DFlash2 drafter, structured output", "decode": 181, "prefill": null, "unit": "tok/s", "context": "16k", "date": "2026-10-13"}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "own MTP head, code; 196k window fits with 8-bit cache", "decode": 116, "prefill": null, "unit": "tok/s", "context": "16k", "date": "2026-10-13"}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "own MTP head, prose", "decode": 70, "prefill": null, "unit": "tok/s", "context": "16k", "date": "2026-10-13"}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "own MTP head, structured output", "decode": 128, "prefill": null, "unit": "tok/s", "context": "16k", "date": "2026-10-13"}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "no drafting, one round", "decode": 42, "prefill": null, "unit": "tok/s", "context": "16k", "date": "2026-10-13"}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "1.6.2, DFlash2, thinking off", "decode": 140.1, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-16"}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "MTP served at 196k, short code; prompts read at 650-740", "decode": 115, "prefill": "650-740", "unit": "tok/s", "context": "196k", "date": null}, + {"model": "Qwen3.8-27B (EXL3 4.00 bpw)", "config": "MTP served at 196k, 141k of context", "decode": 72, "prefill": null, "unit": "tok/s", "context": "141k history", "date": null} + ], + "bugs": [ + {"text": "At temperature 0, drafting changed 7 of 15 DFlash2 replies", "status": "open", "date": "2026-10-13"}, + {"text": "At temperature 0, drafting changed 10 of 15 MTP replies", "status": "open", "date": "2026-10-13"}, + {"text": "MTP setup scored 0 in five of six one-shot pagoda tries", "status": "open", "date": "2026-10-14"} + ], + "doc": "docs/model-research/engines.md#exllamav3-tabbyapi" + }, + { + "id": "tensorfold", + "name": "TensorFold", + "kind": "dense model server with DFlash2 drafting", + "versions": [ + {"version": "0.6.6 (Python engine)", "date": "2026-10-13", "note": "Fastest writer measured for Qwen3.8-27B on the 3090; Python line frozen upstream."}, + {"version": "1.1.0 (upstream)", "date": "2026-10-18", "note": "Reports native RTX 30 support; we have not built or run it."} + ], + "good_at": "Raw decode speed; exact replies at temperature 0", + "watch_out": "Needs a local patch below compute capability 8.9; bf16 cache only", + "multi_session": {"ok": null, "note": "Not measured"}, + "speeds": [ + {"model": "Qwen3.8-27B", "config": "DFlash2 drafter, short code (167-170 across runs)", "decode": 170, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-13"}, + {"model": "Qwen3.8-27B", "config": "DFlash2 drafter, prose (81-84 across runs)", "decode": 84, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-13"}, + {"model": "Qwen3.8-27B", "config": "DFlash2 drafter, structured output (209-211 across runs)", "decode": 211, "prefill": null, "unit": "tok/s", "context": null, "date": "2026-10-13"} + ], + "bugs": [ + {"text": "Needed a local patch to run below compute capability 8.9", "status": "worked around", "date": null}, + {"text": "Cache is bf16 only: 42k window by default, 73k at the edge", "status": "open", "date": null}, + {"text": "Python line is frozen upstream; engine parked, not in the router", "status": "open", "date": null} + ], + "doc": "docs/model-research/engines.md#tensorfold" + }, + { + "id": "tqllm", + "name": "tqLLM", + "kind": "Flash-Next GGUF server (trial)", + "versions": [ + {"version": "1.2.3", "date": "2026-10-22", "note": "Trial with the Flash-Next IQ3_S GGUF and its MTP head against the live llama.cpp entry; first load 23 minutes cold, then 2-3."} + ], + "good_at": "Fast short replies: about 2.5 times llama.cpp on the same GGUF", + "watch_out": "Fails after one out-of-memory prefill; health check still says ok", + "multi_session": {"ok": false, "note": "Failed: 4 of 11 at best; llama.cpp passed 11"}, + "speeds": [ + {"model": "Flash-Next IQ3_S + MTP head", "config": "short outputs, thinking off, code; prefill about 2,000 on a 30k prompt", "decode": 101.7, "prefill": 2000, "unit": "tok/s", "context": "32k", "date": "2026-10-22"}, + {"model": "Flash-Next IQ3_S + MTP head", "config": "short outputs, thinking off, prose", "decode": 79.9, "prefill": null, "unit": "tok/s", "context": "32k", "date": "2026-10-22"}, + {"model": "Flash-Next IQ3_S + MTP head", "config": "short outputs, thinking off, structured output", "decode": 111.8, "prefill": null, "unit": "tok/s", "context": "32k", "date": "2026-10-22"}, + {"model": "Flash-Next IQ3_S + MTP head", "config": "default 262k window, same bench, code", "decode": 105.2, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flash-Next IQ3_S + MTP head", "config": "default 262k window, same bench, prose", "decode": 83.5, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flash-Next IQ3_S + MTP head", "config": "default 262k window, same bench, structured output", "decode": 107.1, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"}, + {"model": "Flash-Next IQ3_S + MTP head", "config": "smoke decode of a 37-token prompt, best 262k arm (not a bench)", "decode": 53.9, "prefill": null, "unit": "tok/s", "context": "262k", "date": "2026-10-22"} + ], + "bugs": [ + {"text": "First prompt short on VRAM for prefill fails; every later request fails", "status": "open", "date": "2026-10-22"}, + {"text": "At 131072 the first failure came on a 289-token prompt", "status": "open", "date": "2026-10-22"}, + {"text": "Health check keeps answering ok after failure, so a router never notices", "status": "open", "date": "2026-10-22"}, + {"text": "Default 262k arm answered long prompts with HTTP 500", "status": "open", "date": "2026-10-22"}, + {"text": "Expert-cache flag had no visible effect on Flash-Next", "status": "open", "date": "2026-10-22"} + ], + "doc": "docs/model-research/engines.md#tqllm" + } + ], + "tuning": [ + { + "id": "vram-tiers", + "title": "Pick the VRAM tier first", + "rule": "Choose how much of the model sits in VRAM first, then the model.", + "metrics": [ + {"label": "Everything on the GPU, typical decode", "before": null, "after": "45-175", "unit": "tok/s"}, + {"label": "A few expert layers in RAM, typical decode", "before": null, "after": "60-95", "unit": "tok/s"}, + {"label": "Experts mostly in RAM, typical decode", "before": null, "after": "3-26", "unit": "tok/s"}, + {"label": "Flint Q4_K_XL decode, 15 to 24 expert layers in RAM", "before": 81.4, "after": 64.2, "unit": "tok/s"}, + {"label": "Flint Q4_K_XL prefill of 11.7k prompt, 15 to 24 layers", "before": 2208, "after": 1624, "unit": "tok/s"}, + {"label": "Ornith-1.5-122B decode, 42 to 40 expert layers on CPU", "before": 13.7, "after": 16.3, "unit": "tok/s"} + ], + "chart": "tiers", + "evidence": "Flint Q4_K_XL, 2026-10-21/22: 81.4, 73.1, 64.2 tok/s at 15, 19, 24 expert layers in RAM.", + "caveat": "The expert cache moved the bottom tier up, not out.", + "doc": "docs/model-research/tuning.md#1-vram-tiers-pick-the-tier-first" + }, + { + "id": "expert-cache", + "title": "Cache hot experts on Flash-Next", + "rule": "For Flash-Next, keep all experts in RAM and cache the hot ones on the GPU.", + "metrics": [ + {"label": "Orcarouter IQ4_XS streamed decode", "before": "12-14", "after": 19, "unit": "tok/s"}, + {"label": "Heretic2 IQ4XS-NGQ4 streamed decode", "before": "11-14", "after": 18, "unit": "tok/s"}, + {"label": "Abliterated s1.5 IQ4_XS streamed decode", "before": "11.5-14.6", "after": "19-21", "unit": "tok/s"}, + {"label": "Swift 1.5 IQ3_XXS streamed decode", "before": "13-17", "after": "23-26", "unit": "tok/s"}, + {"label": "Unpruned IQ3_S streamed decode", "before": "12.5-15", "after": "21-23", "unit": "tok/s"}, + {"label": "Prefill on fresh loads, 16k and 48k", "before": 250, "after": "415-427", "unit": "tok/s"} + ], + "chart": "bars", + "evidence": "2026-10-22, Flash-Next only, five GGUFs at 262k: cache beat layer split on all; live on 12 entries.", + "caveat": "Untested for Ornith and Flint; not yet run through tests 1 and 2.", + "doc": "docs/model-research/tuning.md#2-expert-placement--ncmoe-against--cmoe-plus-the-expert-cache" + }, + { + "id": "kv-precision", + "title": "Use 8-bit KV when it buys room", + "rule": "Use 8-bit KV when it buys context, expert cache or batch; else f16.", + "metrics": [ + {"label": "Mean KL, f16 KV against 8-bit KV plus cache", "before": null, "after": 0.0011, "unit": "KL"}, + {"label": "Mean KL between two loads of the new config (noise floor)", "before": null, "after": 0.0002, "unit": "KL"}, + {"label": "Top-1 agreement, old against new config", "before": null, "after": 98.2, "unit": "%"}, + {"label": "Top-1 agreement between two loads (noise floor)", "before": null, "after": 99.6, "unit": "%"}, + {"label": "Needles found in both configs", "before": null, "after": 8, "unit": "of 8"}, + {"label": "vLLM window, FP8 KV against 4/2-bit KV", "before": 150, "after": 240, "unit": "k tokens"} + ], + "chart": "bars", + "evidence": "2026-10-22, Orcarouter GGUF: f16 KV against 8-bit KV plus cache gives mean KL 0.0011, top-1 98.2 %.", + "caveat": "KV type and cache could not be separated on 24 GB.", + "doc": "docs/model-research/tuning.md#3-kv-precision" + }, + { + "id": "batch-ubatch", + "title": "Raise ubatch until it stops loading", + "rule": "Take the largest -ub that still loads with everything else in place.", + "metrics": [ + {"label": "Ornith prefill, 41 expert layers on CPU, -ub 2048", "before": 116, "after": 392, "unit": "tok/s"}, + {"label": "Ornith decode, same change", "before": 16.0, "after": 16.1, "unit": "tok/s"}, + {"label": "Flint prefill of 11.7k prompt at -ub 2048", "before": null, "after": 5357, "unit": "tok/s"} + ], + "chart": "slope", + "evidence": "Ornith, 41 expert layers on CPU, 2026-10-20: -ub 2048 lifted prefill from 116 to 392 tok/s.", + "caveat": "Too large fails: OrcaSAQ-2 needed -ub 256 at 22.7 of 24 GiB.", + "doc": "docs/model-research/tuning.md#4-batch-and-ubatch" + }, + { + "id": "context-margin", + "title": "Tell the agent a smaller context", + "rule": "Give the agent a context window smaller than the server's by max(10%, 16k).", + "metrics": [ + {"label": "Compaction point on a 150k vLLM server", "before": 71000, "after": 89328, "unit": "tokens"}, + {"label": "Share of window at compaction", "before": 47, "after": 60, "unit": "%"}, + {"label": "Compactions in one 57k vLLM test", "before": null, "after": 55, "unit": "times"}, + {"label": "Compactions with the Pi agent on the 57k setup", "before": null, "after": 8, "unit": "times"} + ], + "chart": "slope", + "evidence": "2026-10-04: a 240k vLLM case died at 213,761 prompt plus 32,000 reply against a 245,760 window.", + "caveat": "Runs before 2026-10-14, all of test 2, used the old window.", + "doc": "docs/model-research/tuning.md#5-context-margin-and-thinking-budget" + }, + { + "id": "thinking-budget", + "title": "Cap thinking inside the reply cap", + "rule": "Set the thinking budget to min(24000, reply cap minus 6000).", + "metrics": [ + {"label": "Round 1 endpoints rescued, reasoning off", "before": null, "after": 6, "unit": "of 9"}, + {"label": "Round 1 endpoints rescued, budget 4096", "before": null, "after": 8, "unit": "of 9"}, + {"label": "Round 1 endpoints rescued, budget 16384", "before": null, "after": 4, "unit": "of 9"}, + {"label": "Length of the one runaway think (OrcaSAQ-2)", "before": null, "after": 87000, "unit": "characters"} + ], + "chart": "bars", + "evidence": "2026-10-04: a cap of 23,961 under a flat 24,000 budget let one 87,000-character think eat the reply.", + "caveat": "A budget stops runaway thinking, not output loops.", + "doc": "docs/model-research/tuning.md#5-context-margin-and-thinking-budget" + }, + { + "id": "maker-sampling", + "title": "Serve the maker's sampling", + "rule": "Serve the maker's recommended sampling on the server; coding agents usually send none.", + "metrics": [ + {"label": "Ternary Bonsai 2 test 1 score, with the makers' agent settings", "before": 2, "after": 5, "unit": "of 5"}, + {"label": "Tiel Coder Q4_K_XL test 2 decode, makers' temperature 0.6", "before": null, "after": 78.6, "unit": "tok/s"}, + {"label": "Tiel Coder Q4_K_XL test 2 duration", "before": null, "after": 27, "unit": "minutes"} + ], + "chart": "slope", + "evidence": "Adopted for every entry 2026-10-09 after loops ruined several runs; Bonsai 2 test 1 rose 2/5 to 5/5.", + "caveat": "Not a cure-all: Devstral Small 2, Qwen3.6-35B, GLM-4.7-Flash still failed.", + "doc": "docs/model-research/tuning.md#6-maker-sampling" + }, + { + "id": "speculative-decoding", + "title": "Draft only when the model fits", + "rule": "Use speculative decoding when the model is on the GPU; check answers stay unchanged.", + "metrics": [ + {"label": "Qwen3.8-27B code, plain against llama.cpp MTP", "before": 42, "after": 93, "unit": "tok/s"}, + {"label": "Qwen3.8-27B code, plain against ExLlamaV3 MTP", "before": 42, "after": 116, "unit": "tok/s"}, + {"label": "Qwen3.8-27B code, plain against ExLlamaV3 DFlash2", "before": 42, "after": 158, "unit": "tok/s"}, + {"label": "Qwen3.8-27B code, plain against TensorFold DFlash2", "before": 42, "after": 170, "unit": "tok/s"}, + {"label": "Gemma 4 26B-A4B with MTP drafter, short code against 141k of history", "before": 267, "after": 123, "unit": "tok/s"}, + {"label": "Ornith-1.5-122B experts in RAM, no draft against MTP with two drafts", "before": 16.3, "after": 16.7, "unit": "tok/s"}, + {"label": "Ornith-1.5-122B, no draft against a DFlash drafter", "before": 16.3, "after": 11.5, "unit": "tok/s"}, + {"label": "Maya-M IQ2_S, no draft against DFlash2 draft", "before": 15.35, "after": 10.7, "unit": "tok/s"} + ], + "chart": "bars", + "evidence": "2026-10-13, Qwen3.8-27B code, one card: 42 tok/s plain, 170 with TensorFold DFlash2; Ornith in RAM gained nothing.", + "caveat": "At temperature 0 drafting changed 7 of 15 DFlash2 replies; TensorFold none.", + "doc": "docs/model-research/tuning.md#7-speculative-decoding-when-it-pays" + }, + { + "id": "warm-cache", + "title": "Warm up and measure twice", + "rule": "Warm up before timing; run each setting twice in opposite orders; never rank on one run.", + "metrics": [ + {"label": "Strata 0.1.42 prompt reading, cold against warm", "before": 44, "after": 1269, "unit": "tok/s"}, + {"label": "Strata 0.1.42 decode, cold against warm round", "before": 13.1, "after": 23.5, "unit": "tok/s"}, + {"label": "Unpruned Flash-Next GGUF, raw benchmark against streamed", "before": 40, "after": "21-23", "unit": "tok/s"}, + {"label": "Prefill, quality-script runs (not reproduced) → fresh load", "before": "85-100", "after": 415, "unit": "tok/s"} + ], + "chart": "slope", + "evidence": "Strata 0.1.42, 2026-10-11: the same prompt read at 44 tok/s cold and 1,269 warm.", + "caveat": "Differences smaller than the second-round speedup were treated as noise.", + "doc": "docs/model-research/tuning.md#8-cold-against-warm-page-cache-and-measuring-twice" + }, + { + "id": "pcie-share", + "title": "Measure the PCIe share setting", + "rule": "Measure an engine's GPU-CPU transfer share on your own card; release defaults can be wrong.", + "metrics": [ + {"label": "Strata 0.1.42 long-answer decode, default against old share", "before": 48, "after": 95, "unit": "tok/s"}, + {"label": "Strata 0.1.42 big-prompt reading", "before": 1177, "after": 1983, "unit": "tok/s"}, + {"label": "Test 1 duration", "before": 94, "after": 51, "unit": "minutes"}, + {"label": "Test 2 duration", "before": 98, "after": 61, "unit": "minutes"}, + {"label": "Test 2 tests passed", "before": 72, "after": 77, "unit": "tests"}, + {"label": "Maya-M decode, share 0.36 against 0.25 (not applied)", "before": 16.0, "after": 17.9, "unit": "tok/s"} + ], + "chart": "slope", + "evidence": "Strata 0.1.42 on Flash-Next, 2026-10-11: older PCIe share doubled long-answer decode, 48 to 95 tok/s.", + "caveat": "Maya-M's +12 % candidate was measured 2026-10-17 and not applied.", + "doc": "docs/model-research/tuning.md#9-pcie-share-and-other-engine-internal-knobs" + }, + { + "id": "engine-choice", + "title": "Choose the engine for the model", + "rule": "Choose the engine for the model class, then tune; short-request speed is not agent readiness.", + "metrics": [ + {"label": "Flash-Next, experts in RAM: llama.cpp 0.4 (2026-10-05)", "before": null, "after": 2.2, "unit": "tok/s"}, + {"label": "Flash-Next, experts in RAM: llama.cpp 0.6 (2026-10-05)", "before": null, "after": 8.9, "unit": "tok/s"}, + {"label": "Flash-Next: Strata (2026-10-04)", "before": null, "after": "30-45", "unit": "tok/s"}, + {"label": "Flash-Next: llama.cpp 0.7.0 with cache, streamed (2026-10-22)", "before": null, "after": "19-26", "unit": "tok/s"}, + {"label": "Flash-Next Gyro-S on Strata 0.2.2, code (2026-10-22)", "before": null, "after": 144, "unit": "tok/s"}, + {"label": "Multi-session checks passed: tqLLM (best) against stock SGLang", "before": 4, "after": 11, "unit": "of 11"} + ], + "chart": "timeline", + "evidence": "Flash-Next with experts in RAM: 2.2 tok/s on llama.cpp 0.4, 30-45 on Strata 0.1 (2026-10-04); Gyro-S on Strata 0.2.2 reached 144 on code (2026-10-22).", + "caveat": "Quants differ across rows; Strata's 144 is Gyro-S on code.", + "doc": "docs/model-research/tuning.md#10-engine-choice" + }, + { + "id": "quality-checks", + "title": "Check a setup before trusting speed", + "rule": "Run quality and multi-session checks on every new engine or settings change before ranking speed.", + "metrics": [ + {"label": "Code decode on the same GGUF, llama.cpp → tqLLM", "before": 40.3, "after": 101.7, "unit": "tok/s"}, + {"label": "Multi-session checks passed, tqLLM (best) → llama.cpp live", "before": 4, "after": 11, "unit": "of 11"}, + {"label": "Multi-session checks passed, Carbon-X at one slot", "before": null, "after": 10, "unit": "of 11"}, + {"label": "Old runs regraded, 2026-10-10", "before": null, "after": 125, "unit": "runs"}, + {"label": "Verdicts changed by the regrade", "before": null, "after": 4, "unit": "verdicts"} + ], + "chart": "bars", + "evidence": "2026-10-22: tqLLM decoded about 2.5 times faster than llama.cpp yet passed at best 4 of 11 multi-session checks.", + "caveat": "A speed result without the probe next to it is a lead, not a pick.", + "doc": "docs/model-research/tuning.md#11-the-quality-checks-we-use" + } + ], + "quality_checks": [ + { + "id": "kl-top1", + "name": "KL and top-1 agreement", + "what": "Greedy logprobs old against new config, same GGUF, cuts of 2k-100k tokens", + "threshold": "Compare with the noise floor: KL 0.0002, top-1 99.6 %", + "example": "Expert cache, 2026-10-22: KL 0.0011, top-1 98.2 %" + }, + { + "id": "needle", + "name": "Needle in a haystack", + "what": "Planted code in filler text, 16k to 250k tokens, room left for thinking", + "threshold": null, + "example": "Stock SGLang needles at 100k, 150k, 190k on two of three builds" + }, + { + "id": "digit-copy", + "name": "Digit-copy probe", + "what": "Copies digit strings in long prompts and finds two needles", + "threshold": "12 of 12 strings and both needles", + "example": "Maya 1.0.4 split one string; 1.1.11 copied 12 of 12" + }, + { + "id": "tool-call", + "name": "Tool-call probe", + "what": "One function call with arguments, parsed", + "threshold": null, + "example": "Passed on every engine until tqLLM's first failure" + }, + { + "id": "multi-session", + "name": "Multi-session probe", + "what": "11 checks: parallel answers, two decoding together, tool calls, long plus short, five concurrent", + "threshold": "11 of 11 (10 of 11 at one slot, by design)", + "example": "llama.cpp live 11 of 11; tqLLM 3, 0 and 4 of 11" + }, + { + "id": "agent-tests", + "name": "Real agent tests", + "what": "Tests 1 to 8; the only check that sees what an agent does", + "threshold": null, + "example": "Gyro-S + MTP test 2: full pass in 24 minutes" + }, + { + "id": "grader-audits", + "name": "Grader audits", + "what": "Regrade old runs when a grader changes", + "threshold": null, + "example": "125 runs regraded 2026-10-10; four verdicts changed" + } + ] +} diff --git a/site/data/history.json b/site/data/history.json index f49e295..8793512 100644 --- a/site/data/history.json +++ b/site/data/history.json @@ -1,4 +1,7 @@ { + "schema": "history/1", + "generated_from": "private bake-off tree, 2026-10-07 to 2026-10-25", + "generated": "2026-10-25 11:07 CDT", "runs_new": [ { "case": "wa-saluki-27b-iq2", @@ -359,6 +362,390 @@ "loc": 9497, "errors_per_kloc": 0.4 } + }, + { + "case": "mx22-orcarouter-fn-uncensored-t1-cli-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 41, + "tests_fail": 0, + "subcmds": 10, + "tps": 19.9, + "minutes": 29, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-22 21:40", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 861, + "errors_per_kloc": 0.0 + } + }, + { + "case": "mx22-orcarouter-fn-uncensored-t2t-rust-mcp-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": null, + "tests_fail": null, + "subcmds": null, + "tps": 13.9, + "minutes": 166, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "partial", + "finished": "2026-10-23 00:28", + "struggle": { + "compiles": 17, + "failed_compiles": 14, + "rust_errors": 60, + "loc": 2340, + "errors_per_kloc": 25.6 + } + }, + { + "case": "mx22-orcarouter-fn-uncensored-setupretry1-t4-bugfix-targets-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 4, + "agent": "OpenCode", + "checks": 4, + "tests_pass": 156, + "tests_fail": 0, + "subcmds": null, + "tps": 17.2, + "minutes": 12, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 00:41", + "struggle": { + "compiles": 5, + "failed_compiles": 2, + "rust_errors": 2, + "loc": 9483, + "errors_per_kloc": 0.3 + } + }, + { + "case": "mx22-orcarouter-fn-uncensored-setupretry1-t4-config-timeouts-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 4, + "agent": "OpenCode", + "checks": 13, + "tests_pass": 170, + "tests_fail": 0, + "subcmds": null, + "tps": 17.4, + "minutes": 28, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 01:10", + "struggle": { + "compiles": 9, + "failed_compiles": 5, + "rust_errors": 9, + "loc": 9888, + "errors_per_kloc": 0.9 + } + }, + { + "case": "mx22-orcarouter-fn-uncensored-setupretry1-t4-feature-filter-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 4, + "agent": "OpenCode", + "checks": 11, + "tests_pass": 159, + "tests_fail": 0, + "subcmds": null, + "tps": 14.9, + "minutes": 12, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 01:23", + "struggle": { + "compiles": 2, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9613, + "errors_per_kloc": 0.1 + } + }, + { + "case": "mx22-flashnext-heretic2-t1-cli-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 36, + "tests_fail": 0, + "subcmds": 11, + "tps": 18.2, + "minutes": 86, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 02:52", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 1436, + "errors_per_kloc": 0.7 + } + }, + { + "case": "mx22-flashnext-heretic2-t2-rust-mcp-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 34, + "tests_fail": 0, + "subcmds": null, + "tps": 17.1, + "minutes": 91, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 04:28", + "struggle": { + "compiles": 23, + "failed_compiles": 9, + "rust_errors": 33, + "loc": 1752, + "errors_per_kloc": 18.8 + } + }, + { + "case": "mx22-flashnext-heretic2-t4-bugfix-targets-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 4, + "agent": "OpenCode", + "checks": 4, + "tests_pass": 156, + "tests_fail": 0, + "subcmds": null, + "tps": 13.9, + "minutes": 10, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 05:27", + "struggle": { + "compiles": 4, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9440, + "errors_per_kloc": 0.1 + } + }, + { + "case": "mx22-flashnext-heretic2-t4-config-timeouts-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 4, + "agent": "OpenCode", + "checks": 13, + "tests_pass": 166, + "tests_fail": 0, + "subcmds": null, + "tps": 17.2, + "minutes": 32, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 06:00", + "struggle": { + "compiles": 3, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9748, + "errors_per_kloc": 0.1 + } + }, + { + "case": "mx22-flashnext-heretic2-t4-feature-filter-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 4, + "agent": "OpenCode", + "checks": 11, + "tests_pass": 159, + "tests_fail": 0, + "subcmds": null, + "tps": 12.0, + "minutes": 15, + "compactions": 0, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 06:15", + "struggle": { + "compiles": 3, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9535, + "errors_per_kloc": 0.1 + } + }, + { + "case": "mx22-flashnext-heretic2-t2t-rust-mcp-r1", + "model": "lcpp6-flashnext-heretic2", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 28, + "tests_fail": 0, + "subcmds": null, + "tps": 19.8, + "minutes": 51, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 07:10", + "struggle": { + "compiles": 13, + "failed_compiles": 4, + "rust_errors": 10, + "loc": 1818, + "errors_per_kloc": 5.5 + } + }, + { + "case": "mx22-orcarouter-fn-uncensored-t2setupretry1-t2-rust-mcp-r1", + "model": "lcpp6-orcarouter-fn-uncensored", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 42, + "tests_fail": 0, + "subcmds": null, + "tps": 14.1, + "minutes": 152, + "compactions": null, + "ctx": 262144, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 09:44", + "struggle": { + "compiles": 44, + "failed_compiles": 20, + "rust_errors": 53, + "loc": 3337, + "errors_per_kloc": 15.9 + } + }, + { + "case": "mx22-ornith-122b-q4kxl-t1-cli-r1", + "model": "lcpp6-ornith-122b-q4kxl", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 48, + "tests_fail": 0, + "subcmds": 9, + "tps": 13.4, + "minutes": 90, + "compactions": null, + "ctx": null, + "kv": null, + "verdict": "pass", + "finished": "2026-10-23 13:49", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 1502, + "errors_per_kloc": 0.0 + } + }, + { + "case": "mx22-ornith-122b-q4kxl-t2-rust-mcp-r1", + "model": "lcpp6-ornith-122b-q4kxl", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 43, + "tests_fail": 0, + "subcmds": null, + "tps": 12.9, + "minutes": 82, + "compactions": null, + "ctx": null, + "kv": null, + "verdict": "pass", + "finished": "2026-10-23 18:18", + "struggle": { + "compiles": 14, + "failed_compiles": 6, + "rust_errors": 12, + "loc": 1648, + "errors_per_kloc": 7.3 + } + }, + { + "case": "mx22-q38-defiance-192k-author-t1-cli-r1", + "model": "lcpp6-q38-defiance-192k-author", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 82, + "tests_fail": 0, + "subcmds": 13, + "tps": 32.5, + "minutes": 34, + "compactions": null, + "ctx": 196608, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 18:53", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 1944, + "errors_per_kloc": 0.0 + } + }, + { + "case": "mx22-q38-defiance-192k-author-t2-rust-mcp-r1", + "model": "lcpp6-q38-defiance-192k-author", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 46, + "tests_fail": 0, + "subcmds": null, + "tps": 22.6, + "minutes": 65, + "compactions": null, + "ctx": 196608, + "kv": "q8_0", + "verdict": "pass", + "finished": "2026-10-23 23:00", + "struggle": { + "compiles": 27, + "failed_compiles": 6, + "rust_errors": 15, + "loc": 2415, + "errors_per_kloc": 6.6 + } } ], "test3_new": [ @@ -387,6 +774,32 @@ "tps": 7.1, "minutes": 9, "finished": "2026-10-17 20:13" + }, + { + "case": "mx22-flashnext-heretic2-setupretry1-t3g-known-good-mcp-r1", + "model": "lcpp6-flashnext-heretic2", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Known-good server. One probe, 12 calls. REST and MCP both 14 of 14. It left the server unchanged and the judgement is right. Rubric 12.", + "tps": 9.5, + "minutes": 15, + "finished": "2026-10-23 11:39" + }, + { + "case": "mx22-flashnext-heretic2-setupretry1-t3o-own-mcp-r1", + "model": "lcpp6-flashnext-heretic2", + "flavor": "3O", + "mcp_facts": "13/14", + "api_facts": "14/14", + "judgement_right": false, + "rubric": 9, + "note": "Own server. MCP 13 of 14, REST 14 of 14. The fixed server builds, initializes, exposes 7 tools and passes 34 tests, but temperature was still missed and the final judgement is wrong. Rubric 9.", + "tps": 11.8, + "minutes": 32, + "finished": "2026-10-23 12:11" } ], "models_new": { @@ -485,6 +898,38 @@ "label": "no", "status": "none" } + }, + "lcpp6-ornith-122b-q4kxl": { + "id": "lcpp6-ornith-122b-q4kxl", + "family": "Ornith 1.5", + "label": "Ornith 1.5 122B-A10B UD-Q4_K_XL · llama.cpp 0.7, experts partly in RAM", + "base": "ornith-ai/Ornith-1.5-122B", + "arch": "MoE 122B-A10B", + "engine": "llama.cpp 0.7", + "quant": "UD-Q4_K_XL", + "kind": "tune", + "uncensored": false, + "note": "MIT. Fine-tune of Qwen3.5-122B-A10B. Thinking on. An offload sweep on 2026-10-20 with 41 expert layers in RAM and 2048-token batches recorded 16.1 tok/s decode and 392.1 tok/s prefill. The agent runs above are slower; context and KV for those runs were not in a spec snapshot.", + "vision": { + "label": "not surveyed", + "status": null + } + }, + "lcpp6-q38-defiance-192k-author": { + "id": "lcpp6-q38-defiance-192k-author", + "family": "Qwen3.8 27B", + "label": "Defiance IQ4_XS · llama.cpp 0.7, 196608 context, q8 KV, author sampler", + "base": "Qwen/Qwen3.8-27B", + "arch": "dense 27B", + "engine": "llama.cpp 0.7", + "quant": "IQ4_XS", + "kind": "uncensored-tune", + "uncensored": true, + "note": "Uncensored tune. Author sampler, no draft weights. This run served 196608 tokens with q8 KV.", + "vision": { + "label": "not surveyed", + "status": null + } } }, "ab_new": [ @@ -597,6 +1042,186 @@ "prefill": 2539.8, "vram": 23862, "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock off, 400-token code prompt", + "decode": 126.4, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock off, 140826-token prompt, seed 41", + "decode": 85.3, + "quote": null, + "prefill": 736, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock off, 140826-token prompt, seed 42", + "decode": 79.1, + "quote": null, + "prefill": 736, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 1, 400-token code prompt", + "decode": 127.9, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 1, 140826-token prompt, seed 41", + "decode": 94.5, + "quote": null, + "prefill": 1053, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 1, 140826-token prompt, seed 42", + "decode": 87.9, + "quote": null, + "prefill": 1053, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 2, 400-token code prompt", + "decode": 127.5, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 2, 140826-token prompt, seed 41", + "decode": 94.4, + "quote": null, + "prefill": 1056, + "vram": null, + "error": "" + }, + { + "model": "qwen3.8-27b-vllm", + "engine": "vLLM (HyperQwen)", + "variant": "Titan, GPU unlock on, round 2, 140826-token prompt, seed 42", + "decode": 87.7, + "quote": null, + "prefill": 1055, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock off, 400-token code prompt", + "decode": 149.1, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock off, 141004-token prompt, seed 41", + "decode": 97.7, + "quote": null, + "prefill": 2201, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock off, 141004-token prompt, seed 42", + "decode": 97.6, + "quote": null, + "prefill": 1549, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 1, 400-token code prompt", + "decode": 148.6, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 1, 141004-token prompt, seed 41", + "decode": 97.7, + "quote": null, + "prefill": 2397, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 1, 141004-token prompt, seed 42", + "decode": 97.9, + "quote": null, + "prefill": 1746, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 2, 400-token code prompt", + "decode": 148.8, + "quote": null, + "prefill": null, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 2, 141004-token prompt, seed 41", + "decode": 97.7, + "quote": null, + "prefill": 2396, + "vram": null, + "error": "" + }, + { + "model": "lcpp6-tiel-coder-35b-t06", + "engine": "llama.cpp 0.7", + "variant": "Titan, GPU unlock on, round 2, 141004-token prompt, seed 42", + "decode": 97.5, + "quote": null, + "prefill": 1745, + "vram": null, + "error": "" } ], "tests": [ @@ -730,6 +1355,34 @@ "title": "Expert cache, Strata 0.2.2, Flint and tqLLM", "what": "Flash-Next sped up on llama.cpp and Strata; a new MoE and a new engine were tried.", "why": "Make the 125B model usable on one card." + }, + { + "date": "2026-10-23", + "era": "Models", + "title": "Ornith 122B and Defiance pass tests 1 and 2", + "what": "Ornith at 13.4 tok/s and Defiance at 32.5 tok/s both cleared the CLI checks and the Rust server tests.", + "why": "Do the new large and uncensored quants finish a real task?" + }, + { + "date": "2026-10-23", + "era": "Flash-Next", + "title": "Heretic 2 passes the code tests", + "what": "Five CLI checks, a 7-tool server with 34 passing tests, and three maintenance tickets. Both one-shots scored 0.", + "why": "An abliterated Flash-Next quant had not been graded on this matrix." + }, + { + "date": "2026-10-23", + "era": "Harness", + "title": "A trust run whose tests did not compile", + "what": "Orca uncensored passed test 1, plain test 2, and three test-4 tickets. Its trust run's tests failed to compile, so 0 and 0 is not a pass.", + "why": "A green protocol line can hide a broken test build." + }, + { + "date": "2026-10-24", + "era": "Engines", + "title": "GPU unlock speeds long prefill", + "what": "On Titan, vLLM Qwen3.8-27B long-prompt prefill rose from 736 to about 1050 tok/s. Short decode stayed near 127 tok/s.", + "why": "Does unlocking the GPU clock help a model that already fits on the card?" } ], "findings": [ @@ -801,7 +1454,7 @@ "date": "2026-10-22", "title": "tqLLM serves Flash-Next fast but not for agents", "kind": "engine", - "summary": "tqLLM 1.2.3 loaded the Flash-Next IQ3_S GGUF and decoded faster than llama.cpp on short benchmarks, but long prompts and several sessions failed.", + "summary": "tqLLM 1.2.3 decodes Flash-Next about 2.5x faster than llama.cpp on the same GGUF, but one prefill out-of-memory leaves every later request failing until a restart.", "settings": null, "metrics": [ { @@ -829,8 +1482,8 @@ "unit": "of 11" } ], - "quality": "Tool calls and a 30k needle passed at 32k; the 262k arm answered long prompts with HTTP 500.", - "caveat": "Short outputs with thinking off flatter the speed; the best 262k arm streamed 54 tok/s and passed 4 of 11 sessions. First load took 23 minutes cold." + "quality": "Tool calls and a 30k needle passed at 32k; at 262k with 8k prefill chunks it found a needle in 248k tokens. At 131072 context it failed on a 289-token prompt, and every later request failed until a restart.", + "caveat": "Short replies with thinking off; any prompt of 256+ tokens can trigger the failure. First load took 23 minutes cold." }, { "date": "2026-10-21", @@ -1253,13 +1906,194 @@ ], "quality": "Standard graders; both tests passed.", "caveat": "Decode figures come from the run log, not a result file. A 1-bit variant failed both tests." + }, + { + "date": "2026-10-24", + "title": "GPU unlock raises long-prompt prefill by 43%", + "kind": "engine", + "summary": "On Titan, vLLM Qwen3.8-27B was measured with GPU unlock off, then on, then on again. A 140826-token prompt prefilled at 736 tok/s off. Unlock rounds were 1053 and 1056 tok/s on seed 41, and 1053 and 1055 tok/s on seed 42. The probe summary calls that a 43.3% and 43.2% gain. Short code decode moved from 126.4 to 127.9 and 127.5 tok/s. The same check on llama.cpp 0.7 with Tiel left short code decode at 149.1, 148.6 and 148.8 tok/s.", + "settings": null, + "metrics": [ + { + "label": "vLLM prefill, 140826 tokens, seed 41", + "before": 736, + "after": 1053, + "unit": "tok/s" + }, + { + "label": "vLLM prefill, seed 41, unlock round 2", + "before": 736, + "after": 1056, + "unit": "tok/s" + }, + { + "label": "vLLM prefill, seed 42, unlock round 2", + "before": 736, + "after": 1055, + "unit": "tok/s" + }, + { + "label": "vLLM decode, 400-token code prompt, round 1", + "before": 126.4, + "after": 127.9, + "unit": "tok/s" + }, + { + "label": "vLLM deep decode, seed 41", + "before": 85.3, + "after": 94.5, + "unit": "tok/s" + }, + { + "label": "Tiel decode, 400-token code prompt, round 1", + "before": 149.1, + "after": 148.6, + "unit": "tok/s" + }, + { + "label": "Tiel prefill, 141004 tokens, seed 41, round 1", + "before": 2201, + "after": 2397, + "unit": "tok/s" + } + ], + "quality": "The needle check passed in every unlock phase, on and off.", + "caveat": "One off-round and two on-rounds, one card. Short decode barely moved. Tiel long-prompt prefill rose less (2201 to 2397 tok/s on seed 41, 1549 to 1746 on seed 42) and its deep decode stayed 97.7 tok/s on seed 41. The llama.cpp version is from the same day's entry spec, not from a spec file inside the probe directory." + }, + { + "date": "2026-10-23", + "title": "Flash-Next Heretic 2 passes the coding tests and fails both one-shots", + "kind": "model", + "summary": "Test 1 scored 5 of 5 checks, 36 tests and 0 failures at 18.2 tok/s. Test 2 built, initialized, exposed 7 tools, called a tool, and passed 34 tests with 0 failures. Three test-4 tickets passed, with 4, 13 and 11 hidden checks. The orrery one-shot scored 0 of 10 because the SVG did not parse. The pagoda one-shot scored 0 of 10 because every sealed frame was only the clear color.", + "settings": null, + "metrics": [ + { + "label": "Test 1 checks", + "before": null, + "after": 5, + "unit": "of 5" + }, + { + "label": "Test 2 tests passed", + "before": null, + "after": 34, + "unit": "tests" + }, + { + "label": "Orrery score", + "before": null, + "after": 0, + "unit": "of 10" + }, + { + "label": "Pagoda score", + "before": null, + "after": 0, + "unit": "of 10" + } + ], + "quality": "Code tests passed. One-shots did not render a usable scene.", + "caveat": "One run each. The trust-variant test 2 exposed 4 tools and passed 28 tests; that is a different case, not a failure of the 7-tool run." + }, + { + "date": "2026-10-23", + "title": "Ornith 122B and Defiance pass tests 1 and 2", + "kind": "model", + "summary": "Ornith UD-Q4_K_XL scored 5 of 5 CLI checks and 48 tests at 13.4 tok/s, then a 7-tool server with 43 tests and 0 failures at 12.9 tok/s. Defiance IQ4_XS at 196608 context scored 5 of 5 checks and 82 tests at 32.5 tok/s, then 7 tools, 46 tests and 0 failures at 22.6 tok/s.", + "settings": null, + "metrics": [ + { + "label": "Ornith test 1 decode", + "before": null, + "after": 13.4, + "unit": "tok/s" + }, + { + "label": "Ornith test 2 tests passed", + "before": null, + "after": 43, + "unit": "tests" + }, + { + "label": "Defiance test 1 decode", + "before": null, + "after": 32.5, + "unit": "tok/s" + }, + { + "label": "Defiance test 2 tests passed", + "before": null, + "after": 46, + "unit": "tests" + } + ], + "quality": "Both protocol suites passed. No new one-shot grade for either model in this window.", + "caveat": "One run each. Ornith has no spec snapshot, so context and KV are unknown. Defiance one-shots finished later and are not graded." + }, + { + "date": "2026-10-23", + "title": "Orca uncensored passes test 4; its trust tests did not compile", + "kind": "model", + "summary": "Test 1 scored 5 of 5 checks and 41 tests. The plain test 2 built, initialized, exposed 9 tools and passed 42 tests. Three test-4 tickets passed (4, 13 and 11 hidden checks). The trust run also built, initialized and called tools (9 tools), but cargo test exited 101 while compiling. The parsed 0 and 0 counts are not a suite pass.", + "settings": null, + "metrics": [ + { + "label": "Test 1 tests passed", + "before": null, + "after": 41, + "unit": "tests" + }, + { + "label": "Plain test 2 tests passed", + "before": null, + "after": 42, + "unit": "tests" + }, + { + "label": "Trust-run cargo test exit", + "before": null, + "after": 101, + "unit": "exit code" + } + ], + "quality": "Coding tickets passed. Trust-run test quality is incomplete, not a pass.", + "caveat": "The plain test-2 grade is in the prerequisite regrade file, not the main grade table. Test 3 for this model is not in the canonical grade table." + }, + { + "date": "2026-10-23", + "title": "Heretic 2 judges a known-good server right and its own server wrong", + "kind": "test", + "summary": "On the known-good server, MCP and REST were both 14 of 14 and the judgement was right (rubric 12). On its own server, REST was 14 of 14 and MCP 13 of 14. The fixed server builds, initializes, exposes 7 tools and passes 34 tests, but the missing fact is temperature and the final judgement is wrong (rubric 9).", + "settings": null, + "metrics": [ + { + "label": "Known-good rubric", + "before": null, + "after": 12, + "unit": "of 12" + }, + { + "label": "Own-server rubric", + "before": null, + "after": 9, + "unit": "of 12" + }, + { + "label": "Own-server MCP facts", + "before": null, + "after": 13, + "unit": "of 14" + } + ], + "quality": "Fact match was high. The own-server trust decision was wrong.", + "caveat": "One run of each flavor. Minutes use the run duration without model-load time." } ], "amend": [ { "date": "2026-10-22", - "title": "An expert cache doubles Flash-Next on one 3090", - "note": "The 13 to 24 tok/s for orca is the best arm of the sweep. With the shipped config the streamed figure is about 19 tok/s (25 to 37 raw).", + "title": "An expert cache makes Flash-Next 1.5x faster on one 3090", + "note": "Raw decode with short outputs is higher (25-37 tok/s on orca).", "extra_metrics": [ { "label": "Decode, heretic2 (streamed)", @@ -1328,7 +2162,5 @@ ], "note": "Later changes are not visible in the files read." } - }, - "schema": "history/1", - "generated_from": "private bake-off tree, 2026-10-07 to 2026-10-22" + } } diff --git a/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-10000.webp b/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-10000.webp new file mode 100644 index 0000000..256aad9 Binary files /dev/null and b/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-10000.webp differ diff --git a/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-1500.webp b/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-1500.webp new file mode 100644 index 0000000..256aad9 Binary files /dev/null and b/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-1500.webp differ diff --git a/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-5000.webp b/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-5000.webp new file mode 100644 index 0000000..256aad9 Binary files /dev/null and b/site/frames/orrery/mx22-flashnext-heretic2-t5-oneshot-orrery-r1/frame-5000.webp differ diff --git a/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-10000.webp b/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-10000.webp new file mode 100644 index 0000000..715d4d9 Binary files /dev/null and b/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-10000.webp differ diff --git a/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-1500.webp b/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-1500.webp new file mode 100644 index 0000000..05764fd Binary files /dev/null and b/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-1500.webp differ diff --git a/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-5000.webp b/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-5000.webp new file mode 100644 index 0000000..b20cf10 Binary files /dev/null and b/site/frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-5000.webp differ diff --git a/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-10000.webp b/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-10000.webp new file mode 100644 index 0000000..0f097a1 Binary files /dev/null and b/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-10000.webp differ diff --git a/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-1500.webp b/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-1500.webp new file mode 100644 index 0000000..c856d2b Binary files /dev/null and b/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-1500.webp differ diff --git a/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-5000.webp b/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-5000.webp new file mode 100644 index 0000000..dcb7b70 Binary files /dev/null and b/site/frames/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/frame-5000.webp differ diff --git a/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-10000.webp b/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-10000.webp new file mode 100644 index 0000000..21c7817 Binary files /dev/null and b/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-10000.webp differ diff --git a/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-1500.webp b/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-1500.webp new file mode 100644 index 0000000..21c7817 Binary files /dev/null and b/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-1500.webp differ diff --git a/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-5000.webp b/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-5000.webp new file mode 100644 index 0000000..21c7817 Binary files /dev/null and b/site/frames/pagoda/mx22-flashnext-heretic2-t6-oneshot-pagoda-r1/frame-5000.webp differ diff --git a/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-10000.webp b/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-10000.webp new file mode 100644 index 0000000..5ecc65c Binary files /dev/null and b/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-10000.webp differ diff --git a/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-1500.webp b/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-1500.webp new file mode 100644 index 0000000..39de992 Binary files /dev/null and b/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-1500.webp differ diff --git a/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-5000.webp b/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-5000.webp new file mode 100644 index 0000000..69fcbd4 Binary files /dev/null and b/site/frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-5000.webp differ diff --git a/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-10000.webp b/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-10000.webp new file mode 100644 index 0000000..95fb92c Binary files /dev/null and b/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-10000.webp differ diff --git a/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-1500.webp b/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-1500.webp new file mode 100644 index 0000000..3121e6f Binary files /dev/null and b/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-1500.webp differ diff --git a/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-5000.webp b/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-5000.webp new file mode 100644 index 0000000..c9259c6 Binary files /dev/null and b/site/frames/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/frame-5000.webp differ diff --git a/site/live/orrery/t5nm1-lcpp-flint-40b-q3kxl/orrery.svg b/site/live/orrery/t5nm1-lcpp-flint-40b-q3kxl/orrery.svg new file mode 100644 index 0000000..3d056bc --- /dev/null +++ b/site/live/orrery/t5nm1-lcpp-flint-40b-q3kxl/orrery.svg @@ -0,0 +1,207 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/site/live/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/orrery.svg b/site/live/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/orrery.svg new file mode 100644 index 0000000..7c27cdc --- /dev/null +++ b/site/live/orrery/t5nm1-lcpp6-ornith-122b-q4kxl/orrery.svg @@ -0,0 +1,184 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BRASS ORRERY · gear ratios after real planetary periods + 1 Earth revolution = 12 s · inner gears drive each arm · terminator = day / night + + \ No newline at end of file diff --git a/site/live/pagoda/t6nm1-lcpp-flint-40b-q3kxl/page.html b/site/live/pagoda/t6nm1-lcpp-flint-40b-q3kxl/page.html new file mode 100644 index 0000000..ffcc4ad --- /dev/null +++ b/site/live/pagoda/t6nm1-lcpp-flint-40b-q3kxl/page.html @@ -0,0 +1,250 @@ + + + + + + Voxel Japanese Pagoda + + + + + + + + diff --git a/site/live/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/page.html b/site/live/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/page.html new file mode 100644 index 0000000..91e25a5 --- /dev/null +++ b/site/live/pagoda/t6nm1-lcpp6-ornith-122b-q4kxl/page.html @@ -0,0 +1,562 @@ + + + + + +Voxel Japanese Pagoda + + + + + + + diff --git a/site/tools/build_data.py b/site/tools/build_data.py index 673fbd2..c88b392 100644 --- a/site/tools/build_data.py +++ b/site/tools/build_data.py @@ -23,6 +23,13 @@ D = json.load(open(dash)) C = json.load(open(os.path.join(SITE, "data/curated.json"))) H = json.load(open(os.path.join(SITE, "data/history.json"))) +# Engines and tuning deep dives, distilled from docs/model-research/{engines,tuning}.md (site/data/deep.json). +DEEP = json.load(open(os.path.join(SITE, "data/deep.json"))) +REPO = "https://github.com/alexderz/agentic-model-testing/blob/main/" +for grp in ("engines", "tuning"): + for e in DEEP.get(grp, []): + if e.get("doc") and not e["doc"].startswith("https://"): + e["doc"] = REPO + e["doc"] for k, m in H.get("models_new", {}).items(): D["models"].setdefault(k, m) D["runs"] = D["runs"] + H.get("runs_new", []) @@ -118,7 +125,8 @@ def oneshots(test, key): hw["weights"] = "read over the network from a NAS, ~180-280 MB/s first load" out = { "schema": "bakeoff-site/1", - "generated": D.get("generated"), + # The newest source: history.json is mined after the dashboard, so its time wins when present. + "generated": H.get("generated") or D.get("generated"), "built": __import__("datetime").datetime.now().astimezone().strftime("%Y-%m-%d %H:%M %Z"), "title": C["title"], "tagline": C["tagline"], "links": C["links"], "hardware": hw, "caveats": D.get("caveats", []), @@ -126,6 +134,7 @@ def oneshots(test, key): "picks": picks, "timeline": D.get("timeline", []), "events": events, "models": models, "runs": runs, "test3": test3, "ab": ab, + "engines": DEEP.get("engines", []), "tuning": DEEP.get("tuning", []), "quality_checks": DEEP.get("quality_checks", []), "oneshots": {"orrery": oneshots(5, "orrery"), "pagoda": oneshots(6, "pagoda"), "agent-orrery": oneshots(8, "agent-orrery"), "agent-pagoda": oneshots(8, "agent-pagoda")}, }