diff --git a/oneshots/orrery/grades.tsv b/oneshots/orrery/grades.tsv index c52af90..605c941 100644 --- a/oneshots/orrery/grades.tsv +++ b/oneshots/orrery/grades.tsv @@ -35,3 +35,5 @@ t5b5-qwen3.8-27b-vllm qwen3.8-27b-vllm 32000 length 32000 361.3 99.3 0 0 0 0 0 0 mx22-flashnext-heretic2-t5-oneshot-orrery-r1 lcpp6-flashnext-heretic2 140000 stop 33547 1513.4 22.3 0 0 0 0 0 0 Final model-supplied standalone SVG is malformed (mismatched tag, line 77). The harness also greedily concatenated two SVG blocks/prose: that packaging error was separated by extracting the final original codeblock verbatim, which still fails XML parsing. Initial valid background-only fragment is not substituted or graded. t5nm1-lcpp6-ornith-122b-q4kxl lcpp6-ornith-122b-q4kxl 258048 stop 29646 1940.1 15.3 1 1 0 1 0 3 Parses and draws, but planets are left at the central origin instead of translated to arm ends. Only gears visibly move; outer markers stay fixed across sealed frames. Nominal period parameters are not functioning planetary orbits. Toothed wheels turn without credible meshing; no visible planetary day/night. t5nm1-lcpp-flint-40b-q3kxl lcpp-flint-40b-q3kxl 258048 stop 30402 299.0 101.8 1 2 0 0 1 4 Renders with detached teeth and shadow halos; motion present; orbital periods 80/38/20/40/160/300s incorrectly order inner planets; no credible meshing pairs; offset black shadows rotate incorrectly. Independent original-source/frame review. +mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1 maya1128-ds4-xs 126976 stop 47380 1819.6 26.1 1 2 2 1 1 7 Score basis: 7/10 on the final code block, 0 as delivered. Final model-supplied code block (after the model's own in the reply) graded verbatim. The harness packaging (first to last ) joined a draft, prose and the final SVG and does not parse (line 3), so as delivered it scores 0. Final block parses and renders: four inner planets only (Mercury, Venus, Earth, Mars) on exact real period ratios (5.78, 14.76, 24, 45.14 s); arms, ring gears and planet spin move between frames; ring gears are dashed circles turning at arbitrary 12/16/20/26 s with unmeshed pinions; night half is a dark half-disc turning at a fixed 5 s, not tied to the sun; title and footer text collide with the outer brass ring. Reply is 54k chars of thinking before the model's and 21k chars of final SVG. Rendered with tools/render_svg.sh, final-block/frames. +mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1 maya1128-ds4-xs 126976 stop 31986 1234.3 26.0 2 2 2 1 2 9 Score basis: 9/10 as delivered. Reply leak did not recur: no in the reply, one HTML document, one SVG, and the harness cut parses (meta.static.parses true). Scored orrery.svg as delivered, frames in frames/. Six planets at exact ratios with Earth = 8 s (1.9264, 4.9216, 8, 15.048, 94.896, 235.656 s); Uranus and Neptune absent. Arms, toothed gears and planet surfaces all move. Gears are 24-tooth symbols but drive periods are 6 to 11 s against a 28 s sun gear, so they do not obey the gear law or drive the orbits (D1). Night gradient is dark on local +x, which stays away from the sun while the surface spins under it (E2). reasoning.txt is still cut (100796 chars, ends mid-token). diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-100.webp b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-100.webp new file mode 100644 index 0000000..495aac3 Binary files /dev/null and b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-100.webp differ diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-2000.webp b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-2000.webp new file mode 100644 index 0000000..18ee3df Binary files /dev/null and b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-2000.webp differ diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-6000.webp b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-6000.webp new file mode 100644 index 0000000..9ee73de Binary files /dev/null and b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frames/frame-6000.webp differ diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/meta.json b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/meta.json new file mode 100644 index 0000000..7ab6c06 --- /dev/null +++ b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/meta.json @@ -0,0 +1,46 @@ +{ + "case": "mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1", + "finish_reason": "stop", + "first_token_s": 4.1, + "max_tokens": 126976, + "model": "maya1128-ds4-xs", + "reasoning_chars": 100796, + "reply_chars": 23352, + "started": "2026-10-26T18:51:46-0500", + "static": { + "svg_found": true, + "parses": true, + "bytes": 21587, + "animateTransform": 19, + "animate": 7, + "animateMotion": 0, + "keyframes": 0, + "script": 0, + "prose_chars_outside_svg": 1765 + }, + "status": "ok", + "test": "5", + "timings": { + "cache_n": 0, + "prompt_n": 80, + "prompt_ms": 4070.9, + "prompt_per_token_ms": 50.886, + "prompt_per_second": 19.7, + "predicted_n": 31986, + "predicted_ms": 1230218.8, + "predicted_per_token_ms": 38.461, + "predicted_per_second": 26.0, + "draft_n": 25569, + "draft_n_accepted": 16433 + }, + "tok_s": 26.0, + "usage": { + "prompt_tokens": 80, + "completion_tokens": 31986, + "total_tokens": 32066, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 1234.3 +} diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/orrery.svg b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/orrery.svg new file mode 100644 index 0000000..96bf6a1 --- /dev/null +++ b/oneshots/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/orrery.svg @@ -0,0 +1,423 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-100.webp b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-100.webp new file mode 100644 index 0000000..5838b0a Binary files /dev/null and b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-100.webp differ diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-12000.webp b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-12000.webp new file mode 100644 index 0000000..22aef8f Binary files /dev/null and b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-12000.webp differ diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-2000.webp b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-2000.webp new file mode 100644 index 0000000..d451a48 Binary files /dev/null and b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-2000.webp differ diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-6000.webp b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-6000.webp new file mode 100644 index 0000000..b3f9db5 Binary files /dev/null and b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frames/frame-6000.webp differ diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/meta.json b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/meta.json new file mode 100644 index 0000000..e2d0a1f --- /dev/null +++ b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/meta.json @@ -0,0 +1,35 @@ +{ + "case": "mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1", + "finish_reason": "stop", + "first_token_s": 4.1, + "max_tokens": 126976, + "model": "maya1128-ds4-xs", + "reasoning_chars": 98010, + "reply_chars": 75809, + "started": "2026-10-26T02:40:50-0500", + "status": "ok", + "test": "5", + "timings": { + "cache_n": 0, + "prompt_n": 80, + "prompt_ms": 4076.9, + "prompt_per_token_ms": 50.961, + "prompt_per_second": 19.6, + "predicted_n": 47380, + "predicted_ms": 1815553.4, + "predicted_per_token_ms": 38.319, + "predicted_per_second": 26.1, + "draft_n": 39271, + "draft_n_accepted": 25900 + }, + "tok_s": 26.1, + "usage": { + "prompt_tokens": 80, + "completion_tokens": 47380, + "total_tokens": 47460, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 1819.6 +} diff --git a/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/orrery.svg b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/orrery.svg new file mode 100644 index 0000000..925ae8b --- /dev/null +++ b/oneshots/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/orrery.svg @@ -0,0 +1,416 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CLOCKWORK ORRERY + REAL RELATIVE PLANETARY SPEEDS + + ORBIT PERIODS: MERCURY 0.241 · VENUS 0.615 · EARTH 1.000 · MARS 1.881 + + + + \ No newline at end of file diff --git a/oneshots/pagoda/grades.tsv b/oneshots/pagoda/grades.tsv index 7f87e46..f3f3462 100644 --- a/oneshots/pagoda/grades.tsv +++ b/oneshots/pagoda/grades.tsv @@ -51,3 +51,5 @@ t6unc1-qwen3.8-27b-uncensored-vllm qwen3.8-27b-uncensored-vllm 140000 stop 86783 mx22-flashnext-heretic2-t6-oneshot-pagoda-r1 lcpp6-flashnext-heretic2 140000 stop 29545 1359.9 21.8 0 0 0 0 0 0 Original animation throws ReferenceError: clock is not defined before scene rendering; all sealed frames contain only the clear-color background. Offline three.js dependency packaging succeeds. Blank-page rule scores every dimension zero; no diagnostic model-code fix was scored. t6nm1-lcpp6-ornith-122b-q4kxl lcpp6-ornith-122b-q4kxl 258048 stop 6653 523.0 15.4 1 2 1 2 2 8 Draws consistently from boxes; five tiers/upturned eaves but full spire is cropped in every frame. Platform, cherry trees, pond and torii visible. Falling petals, shadows and frame-based camera orbit present. t6nm1-lcpp-flint-40b-q3kxl lcpp-flint-40b-q3kxl 258048 stop 15958 145.2 110.1 1 2 1 2 2 8 Draws consistently from boxes; five tiers but complete spire cropped in all three frames; platform pond cherry tree and partly occluded torii visible; petals shadows and frame-based camera orbit present. Offline CDN rewrite is packaging only; no model fix. +mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1 maya1128-ds4-xs 126976 stop 45946 1651.0 27.9 2 2 2 2 2 10 Score basis: 10/10 on the final code block, 0 as delivered. Final code block (after the model's own ) graded verbatim. The harness extraction (first to last ) joined a draft page that has a syntax error ('a0'), 16k chars of self-review prose and the final page; as delivered it shows only the text dump and an Uncaught SyntaxError, so it scores 0. Final page draws cleanly from boxes: five tiers with raised corner eaves and a full spire, stepped stone platform, three cherry trees, pond (partly cropped at the bottom), torii, drifting petals and shadows, frame-based orbit camera (camAngle += dt*0.055, graded from code). Console: PCFSoftShadowMap deprecation only. Offline CDN rewrite is packaging only. Rendered with tools/render_html.sh, final-block/frames. +mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1 maya1128-ds4-xs 126976 stop 30338 1164.4 26.2 2 2 2 2 2 10 Score basis: 10/10 as delivered. Reply leak did not recur: one HTML fence, harness page.html is complete and runs. Scored as delivered, frames in frames/. Draws cleanly from boxes only (BoxGeometry, InstancedMesh). Five storeys with raised corner eaves and a spire, stone platform, cherry trees, pond and a red torii, all visible. Petals move between frames, shadows are on the ground, camera orbit advances with dt in code. Console warnings only (THREE.Clock deprecated, PCFSoftShadowMap fallback), no uncaught error. reasoning.txt is still cut (75521 chars). A stray sits outside the fence and is not in the page. diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-10000.webp b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-10000.webp new file mode 100644 index 0000000..7c36035 Binary files /dev/null and b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-10000.webp differ diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-1500.webp b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-1500.webp new file mode 100644 index 0000000..0b102a2 Binary files /dev/null and b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-1500.webp differ diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-5000.webp b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-5000.webp new file mode 100644 index 0000000..6661ed4 Binary files /dev/null and b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frames/frame-5000.webp differ diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/meta.json b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/meta.json new file mode 100644 index 0000000..a8bc5b2 --- /dev/null +++ b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/meta.json @@ -0,0 +1,49 @@ +{ + "case": "mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1", + "finish_reason": "stop", + "first_token_s": 6.4, + "max_tokens": 126976, + "model": "maya1128-ds4-xs", + "reasoning_chars": 75521, + "reply_chars": 18024, + "started": "2026-10-26T19:12:25-0500", + "static": { + "html_found": true, + "bytes": 17262, + "complete": true, + "three_module": true, + "importmap": true, + "umd_three": false, + "external_urls": [ + "https://cdn.jsdelivr.net/npm/three@0.186.1/build/three.module.js" + ], + "boxes": 4, + "instanced": true, + "prose_chars_outside_html": 764 + }, + "status": "ok", + "test": "6", + "timings": { + "cache_n": 0, + "prompt_n": 176, + "prompt_ms": 6377.9, + "prompt_per_token_ms": 36.238, + "prompt_per_second": 27.6, + "predicted_n": 30338, + "predicted_ms": 1157992.8, + "predicted_per_token_ms": 38.17, + "predicted_per_second": 26.2, + "draft_n": 25196, + "draft_n_accepted": 17644 + }, + "tok_s": 26.2, + "usage": { + "prompt_tokens": 176, + "completion_tokens": 30338, + "total_tokens": 30514, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 1164.4 +} diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/page.html b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/page.html new file mode 100644 index 0000000..78e4577 --- /dev/null +++ b/oneshots/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/page.html @@ -0,0 +1,651 @@ + + + + + + Voxel Pagoda — Cherry Blossom Scene + + + + + + + + diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-10000.webp b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-10000.webp new file mode 100644 index 0000000..a85927f Binary files /dev/null and b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-10000.webp differ diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-1500.webp b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-1500.webp new file mode 100644 index 0000000..4ddc16d Binary files /dev/null and b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-1500.webp differ diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-5000.webp b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-5000.webp new file mode 100644 index 0000000..3f0c66d Binary files /dev/null and b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frames/frame-5000.webp differ diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/meta.json b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/meta.json new file mode 100644 index 0000000..0296ec2 --- /dev/null +++ b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/meta.json @@ -0,0 +1,35 @@ +{ + "case": "mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1", + "finish_reason": "stop", + "first_token_s": 6.0, + "max_tokens": 126976, + "model": "maya1128-ds4-xs", + "reasoning_chars": 71616, + "reply_chars": 63338, + "started": "2026-10-26T03:11:15-0500", + "status": "ok", + "test": "6", + "timings": { + "cache_n": 0, + "prompt_n": 176, + "prompt_ms": 6025.4, + "prompt_per_token_ms": 34.235, + "prompt_per_second": 29.2, + "predicted_n": 45946, + "predicted_ms": 1644949.0, + "predicted_per_token_ms": 35.802, + "predicted_per_second": 27.9, + "draft_n": 39196, + "draft_n_accepted": 30514 + }, + "tok_s": 27.9, + "usage": { + "prompt_tokens": 176, + "completion_tokens": 45946, + "total_tokens": 46122, + "prompt_tokens_details": { + "cached_tokens": 0 + } + }, + "wall_s": 1651.0 +} diff --git a/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/page.html b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/page.html new file mode 100644 index 0000000..c4ba4f8 --- /dev/null +++ b/oneshots/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/page.html @@ -0,0 +1,626 @@ + + + + + +Voxel Pagoda — Cherry Blossom Shrine + + + + + + + \ No newline at end of file diff --git a/site/data/bakeoff.json b/site/data/bakeoff.json index 517a093..97e7393 100644 --- a/site/data/bakeoff.json +++ b/site/data/bakeoff.json @@ -1,7 +1,7 @@ { "schema": "bakeoff-site/1", - "generated": "2026-10-25 11:07 CDT", - "built": "2026-10-25 11:25 CDT", + "generated": "2026-10-26 22:47 CDT", + "built": "2026-10-26 23:07 CDT", "title": "The Bake-off", "tagline": "Which open-weight models can do real agent work on one consumer GPU, how fast, and with what settings.", "links": { @@ -101,6 +101,105 @@ } ], "findings": [ + { + "date": "2026-10-26", + "title": "DeepSeek Maya-XS passes the graded agent tests", + "kind": "test", + "summary": "DeepSeek-V4.1-Flash Maya-XS on Maya 1.1.28 passed every graded agent run. Test 1 was 5 of 5 checks, with 37 tests then 50. Test 2 exposed 8 tools then 7, and passed 45 tests then 19. The planted-docs run exposed 8 tools, passed 31 tests, and hedged: it tried the planted endpoints, then shipped the real ones first. Test 3 was 14 of 14 on both flavors in both rounds, with rubrics 11, 12, 12 and 11, and the judgement was right each time. All three test-4 tickets passed, with 4, 13 and 11 hidden checks and no failed own tests.", + "settings": "Maya 1.1.28, Maya-XS IQ1_S, max context 131072, vision off; DSpark state not recorded per case", + "metrics": [ + { + "label": "Test 1, own tests", + "before": 37, + "after": 50, + "unit": "tests" + }, + { + "label": "Test 2, own tests", + "before": 45, + "after": 19, + "unit": "tests" + }, + { + "label": "Test 1 speed", + "before": 23.2, + "after": 22.6, + "unit": "tok/s" + }, + { + "label": "Test 4 hidden checks", + "before": null, + "after": 28, + "unit": "of 28" + } + ], + "quality": "No graded agent run failed. The planted-docs code kept the real endpoints first. Test 3 judgements were right in both rounds.", + "caveat": "Two runs for tests 1, 2 and 3. One run for the planted-docs test and for test 4. Round-2 planted-docs and test 4 have no recorded grades in this snapshot and are not scored here. Minutes exclude model-load time. The tok/s figure includes tool time. Context is the served max of 131072; these case dirs have no spec snapshot, and the KV type was not recorded." + }, + { + "date": "2026-10-26", + "title": "DeepSeek one-shots: round 1 packaging scored 0, the final block did not", + "kind": "test", + "summary": "Round 1 orrery is 7 of 10 on the final code block (1, 2, 2, 1, 1): four inner planets at the real period ratios, moving arms, and gears that do not mesh. The harness join of that whole reply does not parse, so the packaged file scores 0. Round 1 pagoda is 10 of 10 on the final page and 0 on the harness join, which stops on a syntax error. Round 2 has no reply leak. The orrery file as delivered is 9 of 10 (2, 2, 2, 1, 2): six planets, Uranus and Neptune absent, gears that do not follow the gear law. The pagoda file as delivered is 10 of 10.", + "settings": null, + "metrics": [ + { + "label": "Orrery, final block then delivered file", + "before": 7, + "after": 9, + "unit": "of 10" + }, + { + "label": "Pagoda, final block then delivered file", + "before": 10, + "after": 10, + "unit": "of 10" + }, + { + "label": "Round 1 harness packaging", + "before": null, + "after": 0, + "unit": "of 10" + } + ], + "quality": "Round 2 pages run as delivered. Round 1 scores are of the final block, not of the harness file.", + "caveat": "One run each. Both rounds have gallery frames and live pieces; round 1 shows the graded final block, not the broken harness extract. Decode on the one-shots was 26.1 and 27.9 tok/s, then 26.0 and 26.2." + }, + { + "date": "2026-10-26", + "title": "DeepSeek Maya-XS bench decode is ahead of the agent runs", + "kind": "model", + "summary": "A short bench of the same entry, temperature 0 and 400 tokens, measured 31.0 tok/s on code, 18.9 on prose and 32.2 on structured output. Agent test 1, which includes tool time, was 23.2 tok/s then 22.6.", + "settings": "temperature 0, 400 tokens", + "metrics": [ + { + "label": "Bench decode, code", + "before": null, + "after": 31.0, + "unit": "tok/s" + }, + { + "label": "Bench decode, prose", + "before": null, + "after": 18.9, + "unit": "tok/s" + }, + { + "label": "Bench decode, structured", + "before": null, + "after": 32.2, + "unit": "tok/s" + }, + { + "label": "Agent test 1", + "before": 23.2, + "after": 22.6, + "unit": "tok/s" + } + ], + "quality": "Not an agent grade. The agent runs above are the quality evidence.", + "caveat": "One bench, taken just before round 1. It is not a paired before-and-after." + }, { "date": "2026-10-24", "title": "GPU unlock raises long-prompt prefill by 43%", @@ -1200,6 +1299,20 @@ "title": "GPU unlock speeds long prefill", "what": "On Titan, vLLM Qwen3.8-27B long-prompt prefill rose from 736 to about 1050 tok/s. Short decode stayed near 127 tok/s.", "why": "Does unlocking the GPU clock help a model that already fits on the card?" + }, + { + "date": "2026-10-26", + "era": "Models", + "title": "DeepSeek Maya-XS passes the agent tests", + "what": "On Maya 1.1.28, both rounds passed tests 1, 2 and 3. Round 1 also passed the planted-docs run and all three test-4 tickets.", + "why": "Can a DeepSeek flash pack do the agent work on one GPU?" + }, + { + "date": "2026-10-26", + "era": "Tests", + "title": "DeepSeek one-shots, scored two ways in round 1", + "what": "Round 1 is 7 of 10 and 10 of 10 on the final code block, and 0 on the harness packaging of the same replies. Round 2 is 9 of 10 and 10 of 10 on the file as delivered.", + "why": "Does the picture hold when the reply is a single file?" } ], "events": [], @@ -2723,6 +2836,22 @@ "label": "not surveyed", "status": null } + }, + "maya1128-ds4-xs": { + "id": "maya1128-ds4-xs", + "family": "DeepSeek", + "label": "DeepSeek-V4.1-Flash Maya-XS · Maya 1.1.28, 131k context", + "base": "deepseek-ai/DeepSeek-V4.1-Flash-1017", + "arch": "MoE", + "engine": "Maya 1.1.28", + "quant": "Maya-XS IQ1_S", + "kind": "stock", + "uncensored": false, + "note": "Served at 131072 context with vision off and DSpark state was not recorded per case. Load used 22.3 to 22.5 GB of VRAM. KV type was not recorded.", + "vision": { + "label": "off in the served entry", + "status": null + } } }, "runs": [ @@ -8053,6 +8182,198 @@ "loc": 2415, "errors_per_kloc": 6.6 } + }, + { + "case": "mx22-maya1128-ds4-xs-t1-cli-r1", + "model": "maya1128-ds4-xs", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 37, + "tests_fail": 0, + "subcmds": 11, + "tps": 23.2, + "minutes": 58, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 01:42", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 1647, + "errors_per_kloc": 0.0 + } + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t1-cli-r1", + "model": "maya1128-ds4-xs", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 50, + "tests_fail": 0, + "subcmds": 10, + "tps": 22.6, + "minutes": 71, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 18:07", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 2283, + "errors_per_kloc": 0.4 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t2-rust-mcp-r1", + "model": "maya1128-ds4-xs", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 45, + "tests_fail": 0, + "subcmds": null, + "tps": 24.4, + "minutes": 57, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 02:40", + "struggle": { + "compiles": 5, + "failed_compiles": 2, + "rust_errors": 6, + "loc": 2099, + "errors_per_kloc": 2.9 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t2t-rust-mcp-r1", + "model": "maya1128-ds4-xs", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 31, + "tests_fail": 0, + "subcmds": null, + "tps": 23.3, + "minutes": 53, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 05:24", + "struggle": { + "compiles": 11, + "failed_compiles": 3, + "rust_errors": 4, + "loc": 1937, + "errors_per_kloc": 2.1 + } + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t2-rust-mcp-r1", + "model": "maya1128-ds4-xs", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 19, + "tests_fail": 0, + "subcmds": null, + "tps": 22.8, + "minutes": 43, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 18:51", + "struggle": { + "compiles": 8, + "failed_compiles": 3, + "rust_errors": 10, + "loc": 1611, + "errors_per_kloc": 6.2 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t4-bugfix-targets-r1", + "model": "maya1128-ds4-xs", + "test": 4, + "agent": "OpenCode", + "checks": 4, + "tests_pass": 156, + "tests_fail": 0, + "subcmds": null, + "tps": 16.1, + "minutes": 7, + "compactions": 0, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 05:31", + "struggle": { + "compiles": 2, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 9480, + "errors_per_kloc": 0.0 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t4-config-timeouts-r1", + "model": "maya1128-ds4-xs", + "test": 4, + "agent": "OpenCode", + "checks": 13, + "tests_pass": 167, + "tests_fail": 0, + "subcmds": null, + "tps": 19.9, + "minutes": 25, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 05:57", + "struggle": { + "compiles": 5, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9866, + "errors_per_kloc": 0.1 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t4-feature-filter-r1", + "model": "maya1128-ds4-xs", + "test": 4, + "agent": "OpenCode", + "checks": 11, + "tests_pass": 156, + "tests_fail": 0, + "subcmds": null, + "tps": 17.2, + "minutes": 10, + "compactions": 0, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 06:08", + "struggle": { + "compiles": 3, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9499, + "errors_per_kloc": 0.1 + } } ], "test3": [ @@ -8744,6 +9065,58 @@ "tps": 11.8, "minutes": 32, "finished": "2026-10-23 12:11" + }, + { + "case": "mx22-maya1128-ds4-xs-t3g-known-good-mcp-r1", + "model": "maya1128-ds4-xs", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 11, + "note": "Known-good server. MCP and REST were both 14 of 14 and the judgement was right. Rubric 11. The server was left unchanged. The briefs were large: three probe passes and 41 calls.", + "tps": 11.7, + "minutes": 29, + "finished": "2026-10-26 04:08" + }, + { + "case": "mx22-maya1128-ds4-xs-t3o-own-mcp-r1", + "model": "maya1128-ds4-xs", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Own server. MCP and REST were both 14 of 14 and the judgement was right. Rubric 12. It found a wrong location field, fixed it, and the server builds, initializes, exposes 8 tools and passes 45 tests.", + "tps": 6.5, + "minutes": 22, + "finished": "2026-10-26 04:30" + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t3g-known-good-mcp-r1", + "model": "maya1128-ds4-xs", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Known-good server, second run. MCP and REST were both 14 of 14 and the judgement was right. Rubric 12. One probe field came back false, so it changed the server. The result builds, initializes, exposes 12 tools and passes 155 tests.", + "tps": 11.9, + "minutes": 26, + "finished": "2026-10-26 19:57" + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t3o-own-mcp-r1", + "model": "maya1128-ds4-xs", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 11, + "note": "Own server, second run. MCP and REST were both 14 of 14 and the judgement was right. Rubric 11. The server was already right and was left unchanged. The first probe brief was large enough that the client truncated it.", + "tps": 9.9, + "minutes": 41, + "finished": "2026-10-26 21:31" } ], "ab": [ @@ -11774,6 +12147,55 @@ "frames/orrery/t5nm1-lcpp-flint-40b-q3kxl/frame-10000.webp" ], "live": "live/orrery/t5nm1-lcpp-flint-40b-q3kxl/orrery.svg" + }, + { + "case": "mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1", + "entry": "maya1128-ds4-xs", + "test": 5, + "finish": "stop", + "tokens": 47380.0, + "wall_s": 1819.6, + "tok_s": 26.1, + "points": { + "A": 1.0, + "B": 2.0, + "C": 2.0, + "D": 1.0, + "E": 1.0 + }, + "total": 7.0, + "note": "Score basis: 7/10 on the final code block, 0 as delivered. Final model-supplied code block (after the model's own in the reply) graded verbatim. The harness packaging (first to last ) joined a draft, prose and the final SVG and does not parse (line 3), so as delivered it scores 0. Final block parses and renders: four inner planets only (Mercury, Venus, Earth, Mars) on exact real period ratios (5.78, 14.76, 24, 45.14 s); arms, ring gears and planet spin move between frames; ring gears are dashed circles turning at arbitrary 12/16/20/26 s with unmeshed pinions; night half is a dark half-disc turning at a fixed 5 s, not tied to the sun; title and footer text collide with the outer brass ring. Reply is 54k chars of thinking before the model's and 21k chars of final SVG. Rendered with tools/render_svg.sh, final-block/frames.", + "frames": [ + "frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-100.webp", + "frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-2000.webp", + "frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-6000.webp", + "frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-12000.webp" + ], + "live": "live/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/orrery.svg" + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1", + "entry": "maya1128-ds4-xs", + "test": 5, + "finish": "stop", + "tokens": 31986.0, + "wall_s": 1234.3, + "tok_s": 26.0, + "points": { + "A": 2.0, + "B": 2.0, + "C": 2.0, + "D": 1.0, + "E": 2.0 + }, + "total": 9.0, + "note": "Score basis: 9/10 as delivered. Reply leak did not recur: no in the reply, one HTML document, one SVG, and the harness cut parses (meta.static.parses true). Scored orrery.svg as delivered, frames in frames/. Six planets at exact ratios with Earth = 8 s (1.9264, 4.9216, 8, 15.048, 94.896, 235.656 s); Uranus and Neptune absent. Arms, toothed gears and planet surfaces all move. Gears are 24-tooth symbols but drive periods are 6 to 11 s against a 28 s sun gear, so they do not obey the gear law or drive the orbits (D1). Night gradient is dark on local +x, which stays away from the sun while the surface spins under it (E2). reasoning.txt is still cut (100796 chars, ends mid-token).", + "frames": [ + "frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-100.webp", + "frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-2000.webp", + "frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-6000.webp" + ], + "live": "live/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/orrery.svg" } ], "pagoda": [ @@ -12956,6 +13378,54 @@ "frames/pagoda/t6nm1-lcpp-flint-40b-q3kxl/frame-10000.webp" ], "live": "live/pagoda/t6nm1-lcpp-flint-40b-q3kxl/page.html" + }, + { + "case": "mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1", + "entry": "maya1128-ds4-xs", + "test": 6, + "finish": "stop", + "tokens": 45946.0, + "wall_s": 1651.0, + "tok_s": 27.9, + "points": { + "A": 2.0, + "B": 2.0, + "C": 2.0, + "D": 2.0, + "E": 2.0 + }, + "total": 10.0, + "note": "Score basis: 10/10 on the final code block, 0 as delivered. Final code block (after the model's own ) graded verbatim. The harness extraction (first to last ) joined a draft page that has a syntax error ('a0'), 16k chars of self-review prose and the final page; as delivered it shows only the text dump and an Uncaught SyntaxError, so it scores 0. Final page draws cleanly from boxes: five tiers with raised corner eaves and a full spire, stepped stone platform, three cherry trees, pond (partly cropped at the bottom), torii, drifting petals and shadows, frame-based orbit camera (camAngle += dt*0.055, graded from code). Console: PCFSoftShadowMap deprecation only. Offline CDN rewrite is packaging only. Rendered with tools/render_html.sh, final-block/frames.", + "frames": [ + "frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-1500.webp", + "frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-5000.webp", + "frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-10000.webp" + ], + "live": "live/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/page.html" + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1", + "entry": "maya1128-ds4-xs", + "test": 6, + "finish": "stop", + "tokens": 30338.0, + "wall_s": 1164.4, + "tok_s": 26.2, + "points": { + "A": 2.0, + "B": 2.0, + "C": 2.0, + "D": 2.0, + "E": 2.0 + }, + "total": 10.0, + "note": "Score basis: 10/10 as delivered. Reply leak did not recur: one HTML fence, harness page.html is complete and runs. Scored as delivered, frames in frames/. Draws cleanly from boxes only (BoxGeometry, InstancedMesh). Five storeys with raised corner eaves and a spire, stone platform, cherry trees, pond and a red torii, all visible. Petals move between frames, shadows are on the ground, camera orbit advances with dt in code. Console warnings only (THREE.Clock deprecated, PCFSoftShadowMap fallback), no uncaught error. reasoning.txt is still cut (75521 chars). A stray sits outside the fence and is not in the page.", + "frames": [ + "frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-1500.webp", + "frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-5000.webp", + "frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-10000.webp" + ], + "live": "live/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/page.html" } ], "agent-orrery": [ @@ -13468,12 +13938,12 @@ ] }, "counts": { - "models": 95, - "runs": 222, - "test3": 53, - "oneshots": 106, - "frames": 243, - "pass": 148, - "live": 78 + "models": 96, + "runs": 230, + "test3": 57, + "oneshots": 110, + "frames": 256, + "pass": 156, + "live": 82 } } diff --git a/site/data/history.json b/site/data/history.json index 8793512..2c1ce0c 100644 --- a/site/data/history.json +++ b/site/data/history.json @@ -1,7 +1,7 @@ { "schema": "history/1", - "generated_from": "private bake-off tree, 2026-10-07 to 2026-10-25", - "generated": "2026-10-25 11:07 CDT", + "generated_from": "private bake-off tree, 2026-10-07 to 2026-10-26", + "generated": "2026-10-26 22:47 CDT", "runs_new": [ { "case": "wa-saluki-27b-iq2", @@ -746,6 +746,198 @@ "loc": 2415, "errors_per_kloc": 6.6 } + }, + { + "case": "mx22-maya1128-ds4-xs-t1-cli-r1", + "model": "maya1128-ds4-xs", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 37, + "tests_fail": 0, + "subcmds": 11, + "tps": 23.2, + "minutes": 58, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 01:42", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 1647, + "errors_per_kloc": 0.0 + } + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t1-cli-r1", + "model": "maya1128-ds4-xs", + "test": 1, + "agent": "OpenCode", + "checks": 5, + "tests_pass": 50, + "tests_fail": 0, + "subcmds": 10, + "tps": 22.6, + "minutes": 71, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 18:07", + "struggle": { + "compiles": 0, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 2283, + "errors_per_kloc": 0.4 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t2-rust-mcp-r1", + "model": "maya1128-ds4-xs", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 45, + "tests_fail": 0, + "subcmds": null, + "tps": 24.4, + "minutes": 57, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 02:40", + "struggle": { + "compiles": 5, + "failed_compiles": 2, + "rust_errors": 6, + "loc": 2099, + "errors_per_kloc": 2.9 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t2t-rust-mcp-r1", + "model": "maya1128-ds4-xs", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 31, + "tests_fail": 0, + "subcmds": null, + "tps": 23.3, + "minutes": 53, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 05:24", + "struggle": { + "compiles": 11, + "failed_compiles": 3, + "rust_errors": 4, + "loc": 1937, + "errors_per_kloc": 2.1 + } + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t2-rust-mcp-r1", + "model": "maya1128-ds4-xs", + "test": 2, + "agent": "OpenCode", + "checks": null, + "tests_pass": 19, + "tests_fail": 0, + "subcmds": null, + "tps": 22.8, + "minutes": 43, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 18:51", + "struggle": { + "compiles": 8, + "failed_compiles": 3, + "rust_errors": 10, + "loc": 1611, + "errors_per_kloc": 6.2 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t4-bugfix-targets-r1", + "model": "maya1128-ds4-xs", + "test": 4, + "agent": "OpenCode", + "checks": 4, + "tests_pass": 156, + "tests_fail": 0, + "subcmds": null, + "tps": 16.1, + "minutes": 7, + "compactions": 0, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 05:31", + "struggle": { + "compiles": 2, + "failed_compiles": 0, + "rust_errors": 0, + "loc": 9480, + "errors_per_kloc": 0.0 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t4-config-timeouts-r1", + "model": "maya1128-ds4-xs", + "test": 4, + "agent": "OpenCode", + "checks": 13, + "tests_pass": 167, + "tests_fail": 0, + "subcmds": null, + "tps": 19.9, + "minutes": 25, + "compactions": 1, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 05:57", + "struggle": { + "compiles": 5, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9866, + "errors_per_kloc": 0.1 + } + }, + { + "case": "mx22-maya1128-ds4-xs-t4-feature-filter-r1", + "model": "maya1128-ds4-xs", + "test": 4, + "agent": "OpenCode", + "checks": 11, + "tests_pass": 156, + "tests_fail": 0, + "subcmds": null, + "tps": 17.2, + "minutes": 10, + "compactions": 0, + "ctx": 131072, + "kv": null, + "verdict": "pass", + "finished": "2026-10-26 06:08", + "struggle": { + "compiles": 3, + "failed_compiles": 1, + "rust_errors": 1, + "loc": 9499, + "errors_per_kloc": 0.1 + } } ], "test3_new": [ @@ -800,6 +992,58 @@ "tps": 11.8, "minutes": 32, "finished": "2026-10-23 12:11" + }, + { + "case": "mx22-maya1128-ds4-xs-t3g-known-good-mcp-r1", + "model": "maya1128-ds4-xs", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 11, + "note": "Known-good server. MCP and REST were both 14 of 14 and the judgement was right. Rubric 11. The server was left unchanged. The briefs were large: three probe passes and 41 calls.", + "tps": 11.7, + "minutes": 29, + "finished": "2026-10-26 04:08" + }, + { + "case": "mx22-maya1128-ds4-xs-t3o-own-mcp-r1", + "model": "maya1128-ds4-xs", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Own server. MCP and REST were both 14 of 14 and the judgement was right. Rubric 12. It found a wrong location field, fixed it, and the server builds, initializes, exposes 8 tools and passes 45 tests.", + "tps": 6.5, + "minutes": 22, + "finished": "2026-10-26 04:30" + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t3g-known-good-mcp-r1", + "model": "maya1128-ds4-xs", + "flavor": "3G", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 12, + "note": "Known-good server, second run. MCP and REST were both 14 of 14 and the judgement was right. Rubric 12. One probe field came back false, so it changed the server. The result builds, initializes, exposes 12 tools and passes 155 tests.", + "tps": 11.9, + "minutes": 26, + "finished": "2026-10-26 19:57" + }, + { + "case": "mx22-maya1128-ds4-xs-r2-t3o-own-mcp-r1", + "model": "maya1128-ds4-xs", + "flavor": "3O", + "mcp_facts": "14/14", + "api_facts": "14/14", + "judgement_right": true, + "rubric": 11, + "note": "Own server, second run. MCP and REST were both 14 of 14 and the judgement was right. Rubric 11. The server was already right and was left unchanged. The first probe brief was large enough that the client truncated it.", + "tps": 9.9, + "minutes": 41, + "finished": "2026-10-26 21:31" } ], "models_new": { @@ -930,6 +1174,22 @@ "label": "not surveyed", "status": null } + }, + "maya1128-ds4-xs": { + "id": "maya1128-ds4-xs", + "family": "DeepSeek", + "label": "DeepSeek-V4.1-Flash Maya-XS · Maya 1.1.28, 131k context", + "base": "deepseek-ai/DeepSeek-V4.1-Flash-1017", + "arch": "MoE", + "engine": "Maya 1.1.28", + "quant": "Maya-XS IQ1_S", + "kind": "stock", + "uncensored": false, + "note": "Served at 131072 context with vision off and DSpark state was not recorded per case. Load used 22.3 to 22.5 GB of VRAM. KV type was not recorded.", + "vision": { + "label": "off in the served entry", + "status": null + } } }, "ab_new": [ @@ -1383,6 +1643,20 @@ "title": "GPU unlock speeds long prefill", "what": "On Titan, vLLM Qwen3.8-27B long-prompt prefill rose from 736 to about 1050 tok/s. Short decode stayed near 127 tok/s.", "why": "Does unlocking the GPU clock help a model that already fits on the card?" + }, + { + "date": "2026-10-26", + "era": "Models", + "title": "DeepSeek Maya-XS passes the agent tests", + "what": "On Maya 1.1.28, both rounds passed tests 1, 2 and 3. Round 1 also passed the planted-docs run and all three test-4 tickets.", + "why": "Can a DeepSeek flash pack do the agent work on one GPU?" + }, + { + "date": "2026-10-26", + "era": "Tests", + "title": "DeepSeek one-shots, scored two ways in round 1", + "what": "Round 1 is 7 of 10 and 10 of 10 on the final code block, and 0 on the harness packaging of the same replies. Round 2 is 9 of 10 and 10 of 10 on the file as delivered.", + "why": "Does the picture hold when the reply is a single file?" } ], "findings": [ @@ -2087,6 +2361,105 @@ ], "quality": "Fact match was high. The own-server trust decision was wrong.", "caveat": "One run of each flavor. Minutes use the run duration without model-load time." + }, + { + "date": "2026-10-26", + "title": "DeepSeek Maya-XS passes the graded agent tests", + "kind": "test", + "summary": "DeepSeek-V4.1-Flash Maya-XS on Maya 1.1.28 passed every graded agent run. Test 1 was 5 of 5 checks, with 37 tests then 50. Test 2 exposed 8 tools then 7, and passed 45 tests then 19. The planted-docs run exposed 8 tools, passed 31 tests, and hedged: it tried the planted endpoints, then shipped the real ones first. Test 3 was 14 of 14 on both flavors in both rounds, with rubrics 11, 12, 12 and 11, and the judgement was right each time. All three test-4 tickets passed, with 4, 13 and 11 hidden checks and no failed own tests.", + "settings": "Maya 1.1.28, Maya-XS IQ1_S, max context 131072, vision off; DSpark state not recorded per case", + "metrics": [ + { + "label": "Test 1, own tests", + "before": 37, + "after": 50, + "unit": "tests" + }, + { + "label": "Test 2, own tests", + "before": 45, + "after": 19, + "unit": "tests" + }, + { + "label": "Test 1 speed", + "before": 23.2, + "after": 22.6, + "unit": "tok/s" + }, + { + "label": "Test 4 hidden checks", + "before": null, + "after": 28, + "unit": "of 28" + } + ], + "quality": "No graded agent run failed. The planted-docs code kept the real endpoints first. Test 3 judgements were right in both rounds.", + "caveat": "Two runs for tests 1, 2 and 3. One run for the planted-docs test and for test 4. Round-2 planted-docs and test 4 have no recorded grades in this snapshot and are not scored here. Minutes exclude model-load time. The tok/s figure includes tool time. Context is the served max of 131072; these case dirs have no spec snapshot, and the KV type was not recorded." + }, + { + "date": "2026-10-26", + "title": "DeepSeek one-shots: round 1 packaging scored 0, the final block did not", + "kind": "test", + "summary": "Round 1 orrery is 7 of 10 on the final code block (1, 2, 2, 1, 1): four inner planets at the real period ratios, moving arms, and gears that do not mesh. The harness join of that whole reply does not parse, so the packaged file scores 0. Round 1 pagoda is 10 of 10 on the final page and 0 on the harness join, which stops on a syntax error. Round 2 has no reply leak. The orrery file as delivered is 9 of 10 (2, 2, 2, 1, 2): six planets, Uranus and Neptune absent, gears that do not follow the gear law. The pagoda file as delivered is 10 of 10.", + "settings": null, + "metrics": [ + { + "label": "Orrery, final block then delivered file", + "before": 7, + "after": 9, + "unit": "of 10" + }, + { + "label": "Pagoda, final block then delivered file", + "before": 10, + "after": 10, + "unit": "of 10" + }, + { + "label": "Round 1 harness packaging", + "before": null, + "after": 0, + "unit": "of 10" + } + ], + "quality": "Round 2 pages run as delivered. Round 1 scores are of the final block, not of the harness file.", + "caveat": "One run each. Both rounds have gallery frames and live pieces; round 1 shows the graded final block, not the broken harness extract. Decode on the one-shots was 26.1 and 27.9 tok/s, then 26.0 and 26.2." + }, + { + "date": "2026-10-26", + "title": "DeepSeek Maya-XS bench decode is ahead of the agent runs", + "kind": "model", + "summary": "A short bench of the same entry, temperature 0 and 400 tokens, measured 31.0 tok/s on code, 18.9 on prose and 32.2 on structured output. Agent test 1, which includes tool time, was 23.2 tok/s then 22.6.", + "settings": "temperature 0, 400 tokens", + "metrics": [ + { + "label": "Bench decode, code", + "before": null, + "after": 31.0, + "unit": "tok/s" + }, + { + "label": "Bench decode, prose", + "before": null, + "after": 18.9, + "unit": "tok/s" + }, + { + "label": "Bench decode, structured", + "before": null, + "after": 32.2, + "unit": "tok/s" + }, + { + "label": "Agent test 1", + "before": 23.2, + "after": 22.6, + "unit": "tok/s" + } + ], + "quality": "Not an agent grade. The agent runs above are the quality evidence.", + "caveat": "One bench, taken just before round 1. It is not a paired before-and-after." } ], "amend": [ diff --git a/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-100.webp b/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-100.webp new file mode 100644 index 0000000..495aac3 Binary files /dev/null and b/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-100.webp differ diff --git a/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-2000.webp b/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-2000.webp new file mode 100644 index 0000000..18ee3df Binary files /dev/null and b/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-2000.webp differ diff --git a/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-6000.webp b/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-6000.webp new file mode 100644 index 0000000..9ee73de Binary files /dev/null and b/site/frames/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/frame-6000.webp differ diff --git a/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-100.webp b/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-100.webp new file mode 100644 index 0000000..5838b0a Binary files /dev/null and b/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-100.webp differ diff --git a/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-12000.webp b/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-12000.webp new file mode 100644 index 0000000..22aef8f Binary files /dev/null and b/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-12000.webp differ diff --git a/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-2000.webp b/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-2000.webp new file mode 100644 index 0000000..d451a48 Binary files /dev/null and b/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-2000.webp differ diff --git a/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-6000.webp b/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-6000.webp new file mode 100644 index 0000000..b3f9db5 Binary files /dev/null and b/site/frames/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/frame-6000.webp differ diff --git a/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-10000.webp b/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-10000.webp new file mode 100644 index 0000000..7c36035 Binary files /dev/null and b/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-10000.webp differ diff --git a/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-1500.webp b/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-1500.webp new file mode 100644 index 0000000..0b102a2 Binary files /dev/null and b/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-1500.webp differ diff --git a/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-5000.webp b/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-5000.webp new file mode 100644 index 0000000..6661ed4 Binary files /dev/null and b/site/frames/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/frame-5000.webp differ diff --git a/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-10000.webp b/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-10000.webp new file mode 100644 index 0000000..a85927f Binary files /dev/null and b/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-10000.webp differ diff --git a/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-1500.webp b/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-1500.webp new file mode 100644 index 0000000..4ddc16d Binary files /dev/null and b/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-1500.webp differ diff --git a/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-5000.webp b/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-5000.webp new file mode 100644 index 0000000..3f0c66d Binary files /dev/null and b/site/frames/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/frame-5000.webp differ diff --git a/site/live/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/orrery.svg b/site/live/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/orrery.svg new file mode 100644 index 0000000..96bf6a1 --- /dev/null +++ b/site/live/orrery/mx22-maya1128-ds4-xs-r2-t5-oneshot-orrery-r1/orrery.svg @@ -0,0 +1,423 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/site/live/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/orrery.svg b/site/live/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/orrery.svg new file mode 100644 index 0000000..925ae8b --- /dev/null +++ b/site/live/orrery/mx22-maya1128-ds4-xs-t5-oneshot-orrery-r1/orrery.svg @@ -0,0 +1,416 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CLOCKWORK ORRERY + REAL RELATIVE PLANETARY SPEEDS + + ORBIT PERIODS: MERCURY 0.241 · VENUS 0.615 · EARTH 1.000 · MARS 1.881 + + + + \ No newline at end of file diff --git a/site/live/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/page.html b/site/live/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/page.html new file mode 100644 index 0000000..78e4577 --- /dev/null +++ b/site/live/pagoda/mx22-maya1128-ds4-xs-r2-t6-oneshot-pagoda-r1/page.html @@ -0,0 +1,651 @@ + + + + + + Voxel Pagoda — Cherry Blossom Scene + + + + + + + + diff --git a/site/live/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/page.html b/site/live/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/page.html new file mode 100644 index 0000000..c4ba4f8 --- /dev/null +++ b/site/live/pagoda/mx22-maya1128-ds4-xs-t6-oneshot-pagoda-r1/page.html @@ -0,0 +1,626 @@ + + + + + +Voxel Pagoda — Cherry Blossom Shrine + + + + + + + \ No newline at end of file