diff --git a/README.md b/README.md
index 467380fd36..04cb43a948 100644
--- a/README.md
+++ b/README.md
@@ -173,7 +173,7 @@ Conversions are explicitly marked: their upstream weights need the linked conver
| GLM-5.3-Flash, FP4-Local | `tensorfold pull nvidia/GLM-5.3-Flash-FP4-Local@76348ddb2be2e35e39ecfc00e75dc07703f4f3d5` | Two GB10s at TP=2, or three at TP=3 over a ring of direct links: @Bizuayeu loaded it as stored and served it at both sizes. 165.3 GiB of weights, 13% less than NVFP4's 190.4 GiB. | [Native CUDA](docs/recipes/glm-5.3-flash-cuda.md), in 1.1.5 (#1050). NVIDIA's mixed-precision checkpoint: NVFP4 routed experts with compressed block scales, MXFP8 and 128×128-block FP8 for most of the rest. As close to BF16 as NVFP4 at its smaller size: at TP=2 its NLL is 1.11% above BF16's against NVFP4's 1.37% ([contributed receipt](docs/receipts/glm-fp4local-80517fc3/README.md) on #1050's head, not rerun on main), a difference the per-document intervals in [XSFP4's card](https://huggingface.co/Bizuayeu/GLM-5.3-Flash-XSFP4#measured) do not resolve. |
| GLM-5.3-Flash, XSFP4 | `tensorfold pull Bizuayeu/GLM-5.3-Flash-XSFP4@88736f916f8939607fde8a4b881dcae749f7551d` | Two GB10s at TP=2, or three at TP=3 over a ring of direct links: on main `8086c205c` @Bizuayeu loaded it in 370.7 s at TP=2 (rank 0's model 96.97 GiB at a 524,288-token window) and in 313.8 s at TP=3 (63.60 GiB at 1,048,576), and both served. 165.7 GiB of weights, 0.4 GiB more than FP4-Local's. | [Native CUDA](docs/recipes/glm-5.3-flash-cuda.md), in 1.1.5 through FP4-Local's path. FP4-Local with its routed experts' NVFP4 blocks fitted again from the BF16 weights, a smoothing folded into them, so it runs on the same kernels. Closer to BF16 than FP4-Local and NVFP4 at TP=2 by all three measures in [its card](https://huggingface.co/Bizuayeu/GLM-5.3-Flash-XSFP4#measured), on #1050's code: top-1 agreement 93.26%, NLL 0.85% above and a KL bound of 0.0431, against FP4-Local's 92.82%, 1.11% and 0.0510. |
| GLM-5.3, EXLR8 `home` | `tensorfold pull kindlingai/glm-5.3-exlr8-k2-k3-k4@f459f5d0272a529403dd9b3fd32f771f4250a0d4 --exlr8 home --node 0/2 --dir ~/models/glm-5.3-exlr8-home` | Two M5 Ultra Macs, 256 GB each; about 160 GB resident per Mac at 8k context / four streams. | [Two-Mac recipe](docs/glm-5.3-exlr8.md). Use `--node 1/2` on the other Mac; Thunderbolt 5 and MCDMA required. No CUDA backend. |
-| LFM2.5-24B-A2B, MLX 4 / 6 / 8-bit | `tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-4bit@667408d494bde191c2cb9dd4f4549827816b9275`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-6bit@1deec53eb49571e21cdcbf82fd5839248f07ed88`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-8bit@67ab417dd02d0c9545802a726df668620eae049d` | One Mac; M4 Pro checked. | [Native recipe](docs/recipes/lfm2.5-24b-a2b.md), [release history](CHANGELOG.md). |
+| LFM2.5-24B-A2B, MLX 4 / 6 / 8-bit | `tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-4bit@667408d494bde191c2cb9dd4f4549827816b9275`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-6bit@1deec53eb49571e21cdcbf82fd5839248f07ed88`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-8bit@67ab417dd02d0c9545802a726df668620eae049d` | One Mac; M4 Pro checked. | [Native recipe](docs/recipes/lfm2.5-24b-a2b.md), [release history](CHANGELOG.md). LiquidAI's DSpark drafter with `--drafter`. |
| Qwen3.6-35B-A3B, MLX with MTP | `tensorfold pull TensorFold/Qwen3.6-35B-A3B-MLX-4bit-MTP@81169a9bc511a27c1b4eedb77a2cd98ced431847` | One GB10; 20.9 GB checkpoint. | [Qwen3.6](docs/recipes/qwen3.6-moe.md). Native CUDA; the recipe's Metal path belongs to Python. |
| Gemma 4 31B, MLX 4-bit | `tensorfold pull mlx-community/gemma-4-31B-it-4bit@696d436c404745a59f30e4939a658162b0a9e57f`
`tensorfold pull z-lab/gemma-4-31B-it-DFlash@eabd648301ce28583cc14757912e5e0f84e152e1` | One M5 Mac; M5 Ultra checked. M1-M4 refused. | [Native 31B](docs/recipes/gemma-4-native.md). Name the drafter with `--drafter`; images with `--vision`. |
| Gemma 4 12B, converted MLX 4-bit | `hf download google/gemma-4-12B-it --revision 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 --local-dir ~/models/gemma-4-12b-hf`
`hf download google/gemma-4-12B-it-assistant --revision 46d4c6f13f0ac0ad827b915669b8df9b81c64c51 --local-dir ~/models/gemma-4-12b-assistant` | One M5 Mac; converted text weights 6.3 GB, image weights 100 MB, plus assistant/caches. M1-M4 refused. | [Convert and serve](docs/recipes/gemma-4-12b-native.md). Upstream download needs `hf`; conversion needs Python, mlx and mlx-lm. Serving is native. |
@@ -290,7 +290,7 @@ The binary's `capabilities --json` response lists its supported flags and platfo
| `--loop-guard-stop` | As `--loop-guard`, but end the reply where the guard fires: the reasoning so far, no answer or tool calls, `finish_reason` `content_filter` and a `reasoning_loop` error ([API](docs/api.md)). |
| `--no-drafts` | Produce the plain reference through the same engine. |
| `--drafter DRAFTER` | An external drafter, as a directory or a repo id in the Hugging Face cache, for models that take one: Qwen3.8-27B uses DFlash2 and Gemma 4 31B the drafter named in its recipe. Without one they decode plain. |
-| `--drafter-bits 0\|4` | Drafter weight precision where the model offers a choice: 4 bits (the default), or 0 to keep bf16. Qwen3.8-27B on CUDA takes 4 only. |
+| `--drafter-bits 0\|4\|8` | Drafter weight precision where the model offers a choice: 4 bits (the default), or 0 to keep bf16. Qwen3.8-27B on CUDA takes 4 only. LFM2.5-24B-A2B's DSpark drafter runs bf16 as released (0, the default when not given) or 8-bit codes quantized at load (8); it has no 4-bit mode. |
| `--record DIR` | Write each stream's committed tokens, positions and tapped rows under DIR, for drafter training. CUDA Nemotron only; a single stream runs through the lane rounds, and recording stops below 25 GiB free disk. |
| `--keep-warm SECONDS` | Keep the Metal GPU active while idle for this long after a request, default 900; zero disables it. |
| `--cfg-gamma GAMMA` | Metal Drama speech: semantic guidance strength, finite 1–5; 0/default disables it. Only reference-conditioned requests are guided. Enabled replies report `X-Cfg-Gamma`; codec heads stay conditional. |
diff --git a/docs/kernel-contract.md b/docs/kernel-contract.md
index 1ed43ac146..b04aa6996b 100644
--- a/docs/kernel-contract.md
+++ b/docs/kernel-contract.md
@@ -135,8 +135,10 @@ tile_qmm's pipelines and GLM-5.3-Flash's hc_expand_10.
LFM2.5-24B-A2B's shared rounds: the greedy pick (`lfm2_argmax`) and the conv and attention kernels that run every
stream's rows in one dispatch, each bit-equal to the per-stream kernels it replaces. Four controls edit the source: a tie
to the highest id, a nonfinite logit not flagged, and every stream reading or writing the first stream's state or cache.
-Their launches are in `lfm2_records.zig` with the M1 and M2 limits from `--offline`; `lfm2_attn_segs` reserves its 128
-threads in its source, as Metal cannot build it for M1 or M2 at another count.
+The DSpark drafter's tap copy, Markov scan and pick, draft ids, block attention and confidence logit are there too, with seven more controls.
+Its bf16 linears (few rows, 64x64 tiles) and the Markov scan over a bf16 W2 have four and two more.
+Their launches are in `lfm2_records.zig` with the M1 and M2 limits from `--offline`; `lfm2_attn_segs` and
+`lfm2_attn_block` reserve their 128 threads in their source, as Metal cannot build them for M1 or M2 at another count.
Metal also checks Nemotron's routed experts on the engine's route tables, one row to 128: `tf_xup_w3` and `tf_xdown_w3`
against float64 at tf_load16's bf16 x sums (up's relu2 as an interval over its two bf16 roundings), and the rows2, rows4
and six geometry pipelines of each projection bit-equal to them (two controls: member rows read from one row, and
diff --git a/docs/receipts/lfm2-dspark-review/README.md b/docs/receipts/lfm2-dspark-review/README.md
new file mode 100644
index 0000000000..4a2a6f409a
--- /dev/null
+++ b/docs/receipts/lfm2-dspark-review/README.md
@@ -0,0 +1,273 @@
+# DSpark review fixes on current main
+
+Source `14565fa73`, rebased onto main `d0c9dc9fb`, tested on 2026-10-28.
+Apple M4 Pro, 20 GPU cores, 48 GiB RAM, macOS 26.5.1, Zig 0.17.0,
+Apple Metal compiler 32023.883. Builds target `apple_m1`.
+
+The shared and lone guards from the rejected PRs are absent. The lane core retains
+confidence/grant pricing, hold-cost pricing and `copy_first`. Hold ordering reuses
+one bounded allocation instead of allocating on every round. The initialization
+cleanup releases Metal buffers before their owning list; injected scratch and warm
+allocation failures exercise that path. Config admission refuses rank above 256 and
+nonfinite f32 RoPE theta. The quantizer test widens its size calculation before
+multiplication, and `--drafter` refuses a target restricted to one-row verification.
+
+## Checks
+
+- `zig build test test-kernel-contract test-golden fuzz-cortheon native dev-tools -Dcpu=apple_m1 --summary all`:
+ 749/749 steps succeeded; 8,414 tests passed, 46 skipped, 0 failed.
+- `zig build -Dcpu=apple_m1 --summary all`: 239/239 steps succeeded.
+- `zig build test-lfm2 test-lfm2-dspark -Dcpu=apple_m1 --summary all`:
+ 111/111 tests passed. The final scratch/warm failure test also passed in a subsequent
+ focused run: 51/51 DSpark tests.
+- `zig build test-lfm2-dspark -Dcpu=apple_m1 -Doptimize=debug --summary all`:
+ 51/51 passed, including both injected allocation failures.
+- Debug `zig test` on `dspark_config.zig`: 4/4 passed; on `dspark_quant.zig`: 2/2 passed.
+ The original quantizer test from `2e5966f68` aborted with integer overflow in Debug.
+- `zig fmt --check zig` and `git diff --check`: passed.
+- Lean: 428 findings on main, 428 on this source, 0 new.
+- `tf-kernel-contract --only lfm2`: 11 kernels passed gates 1-4, 0 failures,
+ 0 known findings; all 17 planted controls red, 0 missed; 0 limit problems.
+- `tf-kernel-contract --only tile_qmm`: 6 kernels passed gates 1-4, 0 failures,
+ 0 known findings; 0 limit problems.
+- `tf-lfm2-dspark MODEL DRAFTER 256`: 9/9 drafts equal the independent fp32
+ reference's picks; maximum confidence difference 0.00106; 20 steady drafter
+ rounds had zero host allocations and resizes; exit 0.
+- Completed `tf-lfm2-exact`: 72/72 checks on both M4 Pro and M5 Ultra, zero
+ failures or skipped checks. This supersedes the earlier interrupted run.
+
+## Served equality
+
+Target `LiquidAI/LFM2.5-24B-A2B-MLX-4bit` at
+`667408d494bde191c2cb9dd4f4549827816b9275`, with the local released DSpark weights.
+The drafter safetensors SHA-256 is recorded in the accompanying JSON.
+
+For both default bf16 and explicit `--drafter-bits 8`, serve with `--parallel 16
+--context 4096 --prompt-cache-gib 0 --keep-warm 0`. Run:
+
+```sh
+python3 tools/bench_concurrent.py http://127.0.0.1:18194 review \
+ --tokens 64 --reps 1 --temperatures 0 --levels 1,4,8,16 --alone --serial
+python3 tools/bench_concurrent.py http://127.0.0.1:18194 review \
+ --tokens 64 --reps 1 --temperatures 0 --levels 1,4,8,16 --alone --serial \
+ --prompts tools/bench_prompts.json
+```
+
+Each precision mode passed all 12 cells: eight same-prompt cells (code and chat)
+and four distinct-prompt cells. Concurrent replies equal solo replies and drafted
+solo replies equal `"draft": false` token hashes. Zero unequal replies and zero
+failed requests. [Equality evidence](equality-and-load.json) retains per-request
+concurrent, solo and plain token hashes.
+
+These are short correctness checks, with one repetition, while builds were also
+running. Their timings do not establish the decode performance gate.
+
+## Peak during load
+
+One model process at a time under a model lock. Both servers used the same target
+and bf16 drafter, `--parallel 1 --context 4096 --prompt-cache-gib 0 --keep-warm 0`.
+Read `/memory` after readiness and before sending any request. Its lifetime peak
+agreed with `/usr/bin/time -l` at process exit.
+
+- Exact #1392 head `1bb6e5db2`: 16,036,097,352 bytes, 14.935 GiB.
+- Corrected stack source `14565fa73`: 15,527,930,776 bytes, 14.462 GiB.
+
+The measured reduction during load is 508,166,576 bytes (484.6 MiB). This compares
+the original pre-allocation head with the corrected stack rebased onto current
+main; it is not an isolated comparison of #1393's commit on the same base.
+
+## Extended qualification on the published stack
+
+Source `2dbbc0e8f`, main baseline `d0c9dc9fb`. Additional device: Apple M5 Ultra,
+256 GiB RAM, macOS 27.0, Metal compiler 32023.921.6, Zig 0.17.0.
+[Extended evidence](extended-qualification.json) retains request hashes, counters,
+per-round throughput and latency. Generated text and token arrays are omitted.
+[Reproduction harness](qualification.py) supports `functional`, `long`,
+`performance`, `performance qwen`, and `cold`; set `TF_QUAL_MAIN`,
+`TF_QUAL_CANDIDATE`, `TF_QUAL_MODEL`, `TF_QUAL_DRAFTER`, and `TF_QUAL_OUT`
+for the local installations. Run one model process at a time.
+
+Both devices passed real two-turn resumed-versus-fresh chat equality (3,517 cached
+prompt tokens), 29 seeded sampled equality cases across 1/4/8/16 streams, and
+cancellation/replacement stress with Metal API validation enabled. M4 completed
+143 requests with 84 cancellations at six slots and 252 with 134 at sixteen;
+M5 completed 453 with 247 and 852 with 459 respectively. Zero wrong replies or
+request errors. Server shutdowns were graceful.
+
+M5 plain/drafted serving hashes agree at 4,096, 16,384, 65,536 and 130,816 prompt
+tokens. Independent fp32 drafter checks agree on all nine picks at histories
+65,535 and 130,815; maximum confidence differences 0.00532 and 0.00351.
+Twenty steady rounds had zero host allocations/resizes. M5 also passes the same
+11 LFM2 and six tile_qmm contracts, with all 17 LFM2 planted controls red.
+
+### Performance remains an open issue
+
+Three alternating rounds per arm, 256 generated tokens, 1/4/8/16 streams,
+32-token warmups and uncached prompts. Aggregate decode rate uses the span from
+the earliest first token to the latest last token. The `chat` fixture label in
+the JSON means prose submitted to raw completions, not a templated chat request.
+All cross-arm output hashes agree. These results do **not** clear a no-loss gate.
+
+On M5, drafted versus main-plain median decode changes for code at 1/4/8/16
+streams are +19.44%, -0.96%, -2.20%, -4.23%; for prose they are +35.89%,
+-0.72%, -1.25%, -1.17%. Candidate-plain closely follows main-plain.
+On M4, changes are approximately -6.0%, -13.9%, -5.6%, +0.7% for code and
+-6.4%, -13.4%, -7.8%, -8.0% for prose. M4 also shows variation between plain
+arms, so these measurements do not isolate every source of the loss.
+
+M5 cold prefill medians at 2k/8k/32k/64k tokens are 0.1527/0.6027/2.9244/7.3276
+seconds on main-plain versus 0.1634/0.6327/3.0466/7.5652 with the drafter.
+Prompt counts were checked and cached-token counts were zero.
+
+Other-family Qwen3.8-27B plus DFlash2 was tested at one and four streams with
+three alternating rounds on both devices. M4 results vary with each executable's
+independently measured persistent cost record: fresh records give code changes
+of -7.13%/+3.71% and prose -6.47%/+0.83%. The original records and fresh-record
+runs are both retained. No timings were copied between executables. M5 changes are +0.20%/-0.47% for code and +0.05%/+0.70% for prose.
+No blanket other-family performance pass is claimed.
+
+Two isolated, uncommitted experiments (a larger absorb staging buffer and
+removing forced per-stream copy pricing) did not consistently remove the loss.
+Neither experiment is part of this PR. Further pricing work requires its own
+validation; the rejected shared/lone guards have not been restored.
+
+M1, M2, M3, CUDA and HIP were not rerun for this revision. Focused Metal checks
+are not a full cross-family kernel-contract receipt. All nine GitHub checks passed on `2dbbc0e8f` before this receipt-only update.
+
+### Performance follow-up (`b4deee277`, M4 and M5)
+
+Candidate source `b4deee277` keeps context ingestion on the original bf16 row
+kernel through 32 rows, and uses the existing tiled kernel for held blocks past
+16 rows. It measures each admissible held depth and interpolates neighboring
+measured prices. The hold-price unit test covers a nonlinear timing curve.
+Qwen/DFlash2 dispatch and scheduling are unchanged; only LFM2 supplies this hold
+price table.
+
+`performance-investigation.json` retains all 64 M4 comparison cells, their output
+hashes, source hashes and actual arm order. Four-round median changes against
+the published PR are +1.06%, +0.55%, effectively flat and +0.57% for code at
+1/4/8/16 streams; prose changes are +0.27%, +0.75%, +0.41% and +0.42%.
+The 43.99 tok/s lone-code outlier is retained. The final arm order repeats the
+previous order, so this is not a perfectly balanced four-round experiment.
+All reply hashes agree. Independent fp32 drafter picks agree in all nine cases.
+Focused DSpark, shared-lane equality and native-option checks pass: 49 build
+steps and 56 tests. Formatting, whitespace and harness syntax checks pass.
+These small gains do not establish a no-loss result against main.
+
+An earlier candidate changed context projections as well as held blocks and
+regressed M4 16-stream prose by 8.38%. Preserving context projections removes
+that regression in this comparison, supporting context arithmetic as a cause.
+A separate acceptance-cap scheduler experiment lost roughly 2–3% in some M5
+cells and is excluded. Faster individual holds alone did not guarantee faster
+serving. The exact retained candidate also completed M5 validation, with source hashes
+matching this tree and tested executable hashes retained in the JSON receipt.
+
+
+M5 completed 72 cells in three Latin rotations of main without the external
+DSpark drafter, the published PR and the candidate. Four-round M4 and three-round
+M5 results use the same 256-token requests and aggregate decode metric. The main
+arm still permits suffix copying; it is not the literal `"draft": false` oracle.
+Candidate versus published median changes at 1/4/8/16 streams are -0.26%, +0.04%,
++0.31%, +0.58% for code and -0.04%, -0.02%, +0.06%, +0.72% for prose.
+Against main, code is +21.87%, -0.90%, -2.22%, -3.51%; prose is +36.18%, -0.55%,
+-1.19%, -0.36%. Wider concurrency therefore still does not clear a no-loss gate.
+
+Final M5 LFM2 contracts: 11 pass, zero failures, all 17 planted controls red,
+zero limit problems. Independent FP32 drafter picks agree in all nine cases,
+with zero host allocations/resizes in 20 steady rounds. Resumed/fresh equality,
+29 seeded sampled cases, eight 8-bit cells across 1/4/8/16 streams, and long
+contexts of 65,536 and 130,816 tokens (drafted/undrafted and four concurrent/solo)
+all pass. Cancellation stress completes 423 requests with 234 cancellations at
+six slots and 827 with 445 cancellations at sixteen slots, zero wrong replies
+or request errors. All servers shut down gracefully. Hashes and counters are in
+`performance-investigation.json`. This extends, rather than replaces, the
+historical full forward/cache and qualification receipts above.
+
+After that qualification, main `dd0a9cc5e` was merged to resolve a new CLI
+conflict. The resolution retains main's drafter-name reporting and this stack's
+optional precision default. Native plus the same focused regression targets
+pass: 66/66 build steps, 56/56 tests. Lean compares at 425 findings on both main
+and the merged tree, zero new. Performance and served-GPU receipts above are
+for `b4deee277` before this merge; they are not relabeled as exact merged-head
+measurements.
+
+### Wide-batch calibration (`099455490`)
+
+LFM2 timed single-stream verification windows through 16 rows, but the core may
+verify 32 shared rows. It therefore priced the extra rows by proportional
+extrapolation. This revision measures the shared path with two individually
+qualified windows, and anchors its measured increments to the existing 16-row
+window cost. Per-stream qualification remains 16 rows. The granted-depth rule
+also reuses its first copy-proposer decision: asking again can mutate the
+proposer's silence state and change that same decision. A stateful regression
+reproduces the duplicate-query bug. Qwen/DFlash2's policy is unchanged.
+
+`wide-calibration-candidate.json` retains the exact source and executable hashes,
+checkpoint/tokenizer identities, rendered prompt IDs, all 128 cells, request
+SSE timing spans, output hashes and counters. Four Latin rotations compare main
+`dd0a9cc5e`, published head `68c7435b0`, candidate without the drafter, and
+candidate with bf16 drafting. Every output hash agrees. The plain performance
+arms keep suffix copying enabled; literal `"draft": false` is the separate
+correctness oracle.
+
+Against the published PR, candidate drafted code changes at 1/4/8/16 streams are
++0.48%, **-1.93%**, -0.01%, **+10.96%**; prose changes are +0.05%, -0.42%,
++0.50%, **+8.86%**. At 16 streams it also beats main: 886.46 versus 830.13 tok/s
+for code, and 862.91 versus 798.61 for prose. The task owner accepted the
+four-stream tradeoff for the larger sixteen-stream gain. This is not a universal
+no-loss claim. The candidate without the drafter also loses 2.04% on four-stream
+prose versus main; all four arms are retained, including that result.
+
+M5 final correctness passes all 11 LFM2 contracts, with all 17 planted controls
+red and zero limit problems; 9/9 independent FP32 drafter picks; 20 steady rounds
+with zero host allocations/resizes; resumed/fresh equality with 3,517 reused
+prompt tokens; 29 seeded sampled cases; eight 8-bit cells; and drafted/plain plus
+four-concurrent/solo equality at 65,536 and 130,816 prompt tokens. Six-slot stress
+completes 423 requests with 233 cancellations; sixteen-slot stress completes
+845 with 455 cancellations. Zero wrong replies or request errors; graceful
+shutdowns throughout.
+
+M4 passes the same 11 operator contracts and 17 controls, 9/9 independent drafter
+picks (worst confidence difference 0.00106), and 20 allocation-free steady
+rounds. Its served run stops at the resumed-cache prerequisite: the OS reports
+memory pressure and explicitly disables keeping prompt states, so reused tokens
+remain zero. Its later stress/sampled/8-bit phases did not run. No pressure guard
+was bypassed, and no new M4 serving or throughput qualification is claimed.
+
+Local native/family/shared-lane checks pass 69/69 steps and 117/117 tests. The
+default build passes 239/239 steps. The direct proposer regression, 14 granted
+and 34 copy-policy tests also pass. M5 pre-request, context-4096, one-stream bf16
+load peaks are 15,709,645,744 bytes for published and 15,733,451,768 for candidate
+(+22.7 MiB, one run each). Hot startup increases by roughly one second. Other
+chips/backends and the full cross-family operator gate were not rerun.
+
+`wide-calibration-ablations.json` retains 765 M5 diagnostic cells and 35 interrupted,
+contended M4 cells. Interrupted runs are marked; they are not qualification
+passes. Larger staging, changed copy-rate policies, confidence fallbacks and
+uniform late-hold budgeting were discarded rather than included in this patch.
+
+To reproduce the four-arm comparison, reserve the GPU exclusively and build
+all three checkouts with `zig build native -Dcpu=apple_m1 -Dversion=1.1.8`, then run:
+
+```sh
+TF_QUAL_MAIN=/path/to/main/zig-out/native/bin/tensorfold-native \
+TF_QUAL_PUBLISHED=/path/to/published/zig-out/native/bin/tensorfold-native \
+TF_QUAL_CANDIDATE=/path/to/candidate/zig-out/native/bin/tensorfold-native \
+TF_QUAL_CANDIDATE_ROOT=/path/to/candidate \
+TF_QUAL_DRAFTER=/path/to/LFM2.5-24B-A2B-DSpark \
+TF_QUAL_OUT=/tmp/tf-lfm2-wide-matrix \
+python3 docs/receipts/lfm2-dspark-review/wide_matrix.py
+```
+
+`TF_QUAL_MODEL` overrides the default target snapshot path. The reproducer uses
+only Python's standard library and the sibling `qualification.py` helper.
+
+### Allocation follow-up
+
+Probability fallback now overwrites its existing output buffer when a provider declines. Grafted probabilities fill the draft prefix directly in the final output. Structured-output held tokens are read directly into the final row buffer before grafts. Probability multiplication order, copy rates, parent indexing and decoding policy are unchanged.
+
+For `n` fallback probabilities this saves `8*n` arena bytes. For `d` drafted prefix nodes it saves `8*d` arena bytes plus their copy. With `h` held tokens and `g` graft tokens, grammar row assembly saves `4*(2*h+g)` arena bytes and the held/joined intermediate copies. These are per affected call, not reductions in model weights or a measured serving peak.
+
+The bounded-workspace regression permits only the final probability buffer, denies allocator growth, and covers both declined providers and grafted prefixes. The previous implementation fails it with OutOfMemory; the refactor passes. A grammar regression covers held-token and graft order, including replacing a rejected graft with grammar-generated tokens. The shared-lane/native host checks pass 57/57 steps, 341 tests passed and three skipped. The Metal draft-source fixture now initializes the shared timing count introduced by the calibration change; otherwise its deliberately undefined backend could crash.
+
+The M5 native build passes. An alternating before/after serving comparison at 1/4/8/16 streams is queued behind another qualification's GPU lock. No extra throughput improvement or universal no-regression result is claimed for this allocation-only follow-up yet.
diff --git a/docs/receipts/lfm2-dspark-review/equality-and-load.json b/docs/receipts/lfm2-dspark-review/equality-and-load.json
new file mode 100644
index 0000000000..f66ec3b0d4
--- /dev/null
+++ b/docs/receipts/lfm2-dspark-review/equality-and-load.json
@@ -0,0 +1,4517 @@
+{
+ "source_commit": "14565fa73",
+ "base_commit": "d0c9dc9fb",
+ "target_revision": "667408d494bde191c2cb9dd4f4549827816b9275",
+ "drafter_weights_sha256": "7d06a78556b0b1f386134508f09b00a3a7e0e5e0ec2cc73119229f4ef7c9b299",
+ "tokens": 64,
+ "seed": 1234,
+ "repetitions": 1,
+ "ignore_eos": true,
+ "prompt_cache_gib": 0,
+ "load_comparison": {
+ "before-alloc-bf16-load": {
+ "parallel": 1,
+ "context": 4096,
+ "mode": "bf16",
+ "memory": {
+ "source": "darwin_rusage_info_v4",
+ "physical_footprint_bytes": 15464950680,
+ "lifetime_peak_physical_footprint_bytes": 16036097352
+ }
+ },
+ "candidate-bf16-load": {
+ "parallel": 1,
+ "context": 4096,
+ "mode": "bf16",
+ "memory": {
+ "source": "darwin_rusage_info_v4",
+ "physical_footprint_bytes": 15504747440,
+ "lifetime_peak_physical_footprint_bytes": 15527930776
+ }
+ }
+ },
+ "cells": [
+ {
+ "mode": "bf16",
+ "pool": "same",
+ "prompt": "code",
+ "temperature": 0.0,
+ "streams": 1,
+ "failed": 0,
+ "alone": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "same",
+ "prompt": "code",
+ "temperature": 0.0,
+ "streams": 4,
+ "failed": 0,
+ "alone": {
+ "equal": 4,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "same",
+ "prompt": "code",
+ "temperature": 0.0,
+ "streams": 8,
+ "failed": 0,
+ "alone": {
+ "equal": 8,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "same",
+ "prompt": "code",
+ "temperature": 0.0,
+ "streams": 16,
+ "failed": 0,
+ "alone": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "same",
+ "prompt": "chat",
+ "temperature": 0.0,
+ "streams": 1,
+ "failed": 0,
+ "alone": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "same",
+ "prompt": "chat",
+ "temperature": 0.0,
+ "streams": 4,
+ "failed": 0,
+ "alone": {
+ "equal": 4,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "same",
+ "prompt": "chat",
+ "temperature": 0.0,
+ "streams": 8,
+ "failed": 0,
+ "alone": {
+ "equal": 8,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "same",
+ "prompt": "chat",
+ "temperature": 0.0,
+ "streams": 16,
+ "failed": 0,
+ "alone": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "distinct",
+ "prompt": "distinct",
+ "temperature": 0.0,
+ "streams": 1,
+ "failed": 0,
+ "alone": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "distinct",
+ "prompt": "distinct",
+ "temperature": 0.0,
+ "streams": 4,
+ "failed": 0,
+ "alone": {
+ "equal": 4,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "distinct",
+ "prompt": "distinct",
+ "temperature": 0.0,
+ "streams": 8,
+ "failed": 0,
+ "alone": {
+ "equal": 8,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "bf16",
+ "pool": "distinct",
+ "prompt": "distinct",
+ "temperature": 0.0,
+ "streams": 16,
+ "failed": 0,
+ "alone": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d",
+ "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d",
+ "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d",
+ "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89",
+ "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89",
+ "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89",
+ "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19",
+ "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19",
+ "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19",
+ "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af",
+ "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af",
+ "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af",
+ "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030",
+ "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030",
+ "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030",
+ "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8",
+ "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8",
+ "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8",
+ "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f",
+ "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f",
+ "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f",
+ "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7",
+ "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7",
+ "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7",
+ "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "same",
+ "prompt": "code",
+ "temperature": 0.0,
+ "streams": 1,
+ "failed": 0,
+ "alone": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "same",
+ "prompt": "code",
+ "temperature": 0.0,
+ "streams": 4,
+ "failed": 0,
+ "alone": {
+ "equal": 4,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "same",
+ "prompt": "code",
+ "temperature": 0.0,
+ "streams": 8,
+ "failed": 0,
+ "alone": {
+ "equal": 8,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "same",
+ "prompt": "code",
+ "temperature": 0.0,
+ "streams": 16,
+ "failed": 0,
+ "alone": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab",
+ "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "same",
+ "prompt": "chat",
+ "temperature": 0.0,
+ "streams": 1,
+ "failed": 0,
+ "alone": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "same",
+ "prompt": "chat",
+ "temperature": 0.0,
+ "streams": 4,
+ "failed": 0,
+ "alone": {
+ "equal": 4,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "same",
+ "prompt": "chat",
+ "temperature": 0.0,
+ "streams": 8,
+ "failed": 0,
+ "alone": {
+ "equal": 8,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "same",
+ "prompt": "chat",
+ "temperature": 0.0,
+ "streams": 16,
+ "failed": 0,
+ "alone": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a",
+ "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "distinct",
+ "prompt": "distinct",
+ "temperature": 0.0,
+ "streams": 1,
+ "failed": 0,
+ "alone": {
+ "equal": 1,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "distinct",
+ "prompt": "distinct",
+ "temperature": 0.0,
+ "streams": 4,
+ "failed": 0,
+ "alone": {
+ "equal": 4,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "distinct",
+ "prompt": "distinct",
+ "temperature": 0.0,
+ "streams": 8,
+ "failed": 0,
+ "alone": {
+ "equal": 8,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ },
+ {
+ "mode": "8bit",
+ "pool": "distinct",
+ "prompt": "distinct",
+ "temperature": 0.0,
+ "streams": 16,
+ "failed": 0,
+ "alone": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "serial": {
+ "equal": 16,
+ "unequal": 0,
+ "failed": 0
+ },
+ "token_hash_comparisons": [
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0",
+ "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f",
+ "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89",
+ "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10",
+ "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458",
+ "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5",
+ "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c",
+ "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55",
+ "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d",
+ "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d",
+ "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d",
+ "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89",
+ "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89",
+ "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89",
+ "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19",
+ "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19",
+ "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19",
+ "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af",
+ "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af",
+ "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af",
+ "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030",
+ "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030",
+ "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030",
+ "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8",
+ "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8",
+ "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8",
+ "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f",
+ "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f",
+ "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f",
+ "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde",
+ "complete": true,
+ "error": null
+ }
+ },
+ {
+ "concurrent": {
+ "tokens": 64,
+ "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7",
+ "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06",
+ "complete": true,
+ "error": null
+ },
+ "alone": {
+ "tokens": 64,
+ "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7",
+ "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06",
+ "complete": true,
+ "error": null
+ },
+ "serial": {
+ "tokens": 64,
+ "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7",
+ "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06",
+ "complete": true,
+ "error": null
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/docs/receipts/lfm2-dspark-review/extended-qualification.json b/docs/receipts/lfm2-dspark-review/extended-qualification.json
new file mode 100644
index 0000000000..db8d93df99
--- /dev/null
+++ b/docs/receipts/lfm2-dspark-review/extended-qualification.json
@@ -0,0 +1,53517 @@
+{
+ "source": "2dbbc0e8f5bbadf1a33351769439026f318ddb8f",
+ "main": "d0c9dc9fb",
+ "date": "2026-10-28",
+ "devices": {
+ "M4 Pro": {
+ "functional": {
+ "resumed": {
+ "usage": {
+ "prompt_tokens": 3605,
+ "completion_tokens": 64,
+ "total_tokens": 3669,
+ "prompt_tokens_details": {
+ "cached_tokens": 3517
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.39393036340272,
+ "seconds": 0.592139042,
+ "prefill_seconds": 0.181146458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.181769458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a064a0a2711c",
+ "token_sha256": "a064a0a2711ce3e5053bd664c3540fc534b5f6044a0aa69b70bce596081d429e",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 31,
+ "drafted": 55,
+ "accepted": 33,
+ "acceptance_rate": 0.6,
+ "tokens_per_round": 2.064516129032258
+ },
+ "wall": 0.7745752497576177
+ },
+ "resume_request": {
+ "messages": [
+ {
+ "role": "user"
+ },
+ {
+ "role": "assistant"
+ },
+ {
+ "role": "user"
+ }
+ ],
+ "max_tokens": 64
+ },
+ "stress_reference_hashes": [
+ {
+ "prompt": "quicksort-raw",
+ "sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0"
+ },
+ {
+ "prompt": "rust-reverse-words",
+ "sha256": "d9499a1af105f9d0477f4fda4f2afde556c782008fe552d4a7cd0ebf71e80101"
+ },
+ {
+ "prompt": "multiply-17-23",
+ "sha256": "3ac253c3ae100e4ca9328988d95e9324ab6de6f85787f191d13c718bfae1f4b4"
+ },
+ {
+ "prompt": "sleep-tips",
+ "sha256": "076c7d4d0802aa0ee09b60f396deab8a6cfc5683e2f17b6b90f587b410d20ba5"
+ },
+ {
+ "prompt": "js-debounce-raw",
+ "sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458"
+ },
+ {
+ "prompt": "linear-equation",
+ "sha256": "9277e1d182a9cfc39ec6df2bb5f33c9acb4ca3db6922f5113769afeb3c2ee38b"
+ },
+ {
+ "prompt": "fishing-village",
+ "sha256": "522136a9089e45996543679366846beba531361185bdc02baa0f336231b8848e"
+ },
+ {
+ "prompt": "sql-left-join",
+ "sha256": "45afcf2bc8f7bdc6b1a1ae93f7a06c53e7a98db5e4d9f283753159d8e2707a74"
+ }
+ ],
+ "stress": [
+ {
+ "slots": 6,
+ "seconds": 60,
+ "completed": 143,
+ "cancelled": 84,
+ "wrong": 0,
+ "errors": []
+ },
+ {
+ "slots": 16,
+ "seconds": 90,
+ "completed": 252,
+ "cancelled": 134,
+ "wrong": 0,
+ "errors": []
+ }
+ ],
+ "sampled": [
+ {
+ "streams": 1,
+ "hashes": [
+ [
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c",
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c",
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c"
+ ]
+ ]
+ },
+ {
+ "streams": 4,
+ "hashes": [
+ [
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c",
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c",
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c"
+ ],
+ [
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1",
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1",
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1"
+ ],
+ [
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d",
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d",
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d"
+ ],
+ [
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738",
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738",
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738"
+ ]
+ ]
+ },
+ {
+ "streams": 8,
+ "hashes": [
+ [
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c",
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c",
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c"
+ ],
+ [
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1",
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1",
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1"
+ ],
+ [
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d",
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d",
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d"
+ ],
+ [
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738",
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738",
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738"
+ ],
+ [
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366"
+ ],
+ [
+ "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf",
+ "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf",
+ "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf"
+ ],
+ [
+ "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e",
+ "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e",
+ "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e"
+ ],
+ [
+ "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb",
+ "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb",
+ "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb"
+ ]
+ ]
+ },
+ {
+ "streams": 16,
+ "hashes": [
+ [
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c",
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c",
+ "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c"
+ ],
+ [
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1",
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1",
+ "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1"
+ ],
+ [
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d",
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d",
+ "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d"
+ ],
+ [
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738",
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738",
+ "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738"
+ ],
+ [
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366"
+ ],
+ [
+ "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf",
+ "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf",
+ "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf"
+ ],
+ [
+ "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e",
+ "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e",
+ "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e"
+ ],
+ [
+ "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb",
+ "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb",
+ "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb"
+ ],
+ [
+ "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c",
+ "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c",
+ "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c"
+ ],
+ [
+ "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b",
+ "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b",
+ "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b"
+ ],
+ [
+ "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2",
+ "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2",
+ "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2"
+ ],
+ [
+ "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3",
+ "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3",
+ "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3"
+ ],
+ [
+ "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de",
+ "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de",
+ "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de"
+ ],
+ [
+ "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74",
+ "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74",
+ "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74"
+ ],
+ [
+ "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8",
+ "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8",
+ "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8"
+ ],
+ [
+ "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8",
+ "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8",
+ "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8"
+ ]
+ ]
+ }
+ ],
+ "fresh": {
+ "usage": {
+ "prompt_tokens": 3605,
+ "completion_tokens": 64,
+ "total_tokens": 3669,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 94.76795634919337,
+ "seconds": 0.664781667,
+ "prefill_seconds": 4.404813625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 4.405911625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a064a0a2711c",
+ "token_sha256": "a064a0a2711ce3e5053bd664c3540fc534b5f6044a0aa69b70bce596081d429e",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 30,
+ "drafted": 63,
+ "accepted": 34,
+ "acceptance_rate": 0.5396825396825397,
+ "tokens_per_round": 2.1333333333333333
+ },
+ "wall": 5.071483042091131
+ },
+ "resume_equal": true
+ },
+ "performance-lfm2": [
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 134.9239307001102,
+ "ttft_max": 0.08557750005275011,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 134.91792528327625,
+ "seconds": 1.890037958,
+ "prefill_seconds": 0.085051458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.085222875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 204,
+ "drafted": 90,
+ "accepted": 51,
+ "acceptance_rate": 0.5666666666666667,
+ "tokens_per_round": 1.2549019607843137
+ },
+ "sent": 3605043.487071625,
+ "first": 3605043.572649125,
+ "last": 3605045.462602958,
+ "chunks": 205
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 228.88303862287736,
+ "ttft_max": 0.3615560829639435,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 59.630612906824005,
+ "seconds": 4.276327,
+ "prefill_seconds": 0.091884916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.267615917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605045.4638635,
+ "first": 3605045.732146916,
+ "last": 3605050.008385625,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 57.22032523118682,
+ "seconds": 4.456458417,
+ "prefill_seconds": 0.087317125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.087474584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605045.463993666,
+ "first": 3605045.551974833,
+ "last": 3605050.008344083,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 60.960532403069806,
+ "seconds": 4.183034333,
+ "prefill_seconds": 0.093315625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.360946209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605045.46392825,
+ "first": 3605045.825484333,
+ "last": 3605050.008399416,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 58.37623247178504,
+ "seconds": 4.368216125,
+ "prefill_seconds": 0.088234584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.175705708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605045.463787041,
+ "first": 3605045.64022925,
+ "last": 3605050.008372833,
+ "chunks": 216
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 266.03897119309573,
+ "ttft_max": 0.7605068329721689,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.09353827701317,
+ "seconds": 7.479423166,
+ "prefill_seconds": 0.0945975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2810115,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605050.009457791,
+ "first": 3605050.291337958,
+ "last": 3605057.771036541,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.42370539324265,
+ "seconds": 7.000935167,
+ "prefill_seconds": 0.096240334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.759415667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605050.009511625,
+ "first": 3605050.770018458,
+ "last": 3605057.770923125,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.929790863783346,
+ "seconds": 7.097174625,
+ "prefill_seconds": 0.096306334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.663202458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605050.009567791,
+ "first": 3605050.673718083,
+ "last": 3605057.771226,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.53131103085494,
+ "seconds": 7.384602333,
+ "prefill_seconds": 0.094837,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.375764167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605050.009616541,
+ "first": 3605050.38616375,
+ "last": 3605057.770763208,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.66776444448369,
+ "seconds": 7.574010458,
+ "prefill_seconds": 0.093538167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.186413208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605050.00977625,
+ "first": 3605050.196752333,
+ "last": 3605057.770742291,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.98275518635287,
+ "seconds": 7.289305792,
+ "prefill_seconds": 0.095362542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.471108958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605050.009734708,
+ "first": 3605050.481606,
+ "last": 3605057.771092125,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.25701120968319,
+ "seconds": 7.667556125,
+ "prefill_seconds": 0.092666459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0928595,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605050.009661,
+ "first": 3605050.1031765,
+ "last": 3605057.770667708,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.44872150295576,
+ "seconds": 7.193489333,
+ "prefill_seconds": 0.095840875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.566871667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605050.0093875,
+ "first": 3605050.577402208,
+ "last": 3605057.770988958,
+ "chunks": 229
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 311.1771970060855,
+ "ttft_max": 1.4885342502966523,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.010677199833463,
+ "seconds": 12.743196917,
+ "prefill_seconds": 0.092467916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.458635584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774299166,
+ "first": 3605058.23469325,
+ "last": 3605070.977982791,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.157114695311563,
+ "seconds": 12.650620084,
+ "prefill_seconds": 0.092833334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.551451666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.77475,
+ "first": 3605058.327514458,
+ "last": 3605070.978081708,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.61350805080629,
+ "seconds": 12.370529042,
+ "prefill_seconds": 0.09319175,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.831520625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.7743725,
+ "first": 3605058.607713041,
+ "last": 3605070.978289375,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.306786367739754,
+ "seconds": 12.557378375,
+ "prefill_seconds": 0.093261458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.644682167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.775298666,
+ "first": 3605058.420757,
+ "last": 3605070.978255583,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.585725861822993,
+ "seconds": 13.019685959,
+ "prefill_seconds": 0.091256791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.18242575,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774426458,
+ "first": 3605057.958124208,
+ "last": 3605070.978151041,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.44945779897582,
+ "seconds": 13.110905334,
+ "prefill_seconds": 0.090963583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.091134833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774480916,
+ "first": 3605057.866821083,
+ "last": 3605070.977915166,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.77102097803939,
+ "seconds": 12.276719583,
+ "prefill_seconds": 0.09383925,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.925197417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.775256541,
+ "first": 3605058.701541375,
+ "last": 3605070.978314,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.459505653813892,
+ "seconds": 12.46364425,
+ "prefill_seconds": 0.093743833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.738352916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774827583,
+ "first": 3605058.514517041,
+ "last": 3605070.978100291,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.86659477677835,
+ "seconds": 12.835616917,
+ "prefill_seconds": 0.092491125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.366140709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774540583,
+ "first": 3605058.1421745,
+ "last": 3605070.978321666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.724519686382095,
+ "seconds": 12.928071459,
+ "prefill_seconds": 0.091558708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.273895042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774587541,
+ "first": 3605058.049665916,
+ "last": 3605070.978064916,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.424400573077474,
+ "seconds": 11.902316666,
+ "prefill_seconds": 0.09373375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.299457542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774637,
+ "first": 3605059.076021416,
+ "last": 3605070.978304916,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.767101066869646,
+ "seconds": 11.714927,
+ "prefill_seconds": 0.0936205,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.486714584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774866916,
+ "first": 3605059.263401166,
+ "last": 3605070.978264833,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.930350667736132,
+ "seconds": 12.183264583,
+ "prefill_seconds": 0.093447667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.01861425,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774904541,
+ "first": 3605058.795012791,
+ "last": 3605070.978209041,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.092359658728103,
+ "seconds": 12.08968575,
+ "prefill_seconds": 0.09357025,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.112186208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.77468375,
+ "first": 3605058.888590125,
+ "last": 3605070.978239833,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.594453198550667,
+ "seconds": 11.808587958,
+ "prefill_seconds": 0.093753917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.3930715,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774950875,
+ "first": 3605059.169768458,
+ "last": 3605070.978273,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.25698395196336,
+ "seconds": 11.996057417,
+ "prefill_seconds": 0.093664167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.205769666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605057.774183791,
+ "first": 3605058.9823115,
+ "last": 3605070.978227666,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 157.14330724343966,
+ "ttft_max": 0.09867354203015566,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 157.13458006784825,
+ "seconds": 1.62281275,
+ "prefill_seconds": 0.098100917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098283125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 144,
+ "drafted": 180,
+ "accepted": 111,
+ "acceptance_rate": 0.6166666666666667,
+ "tokens_per_round": 1.7777777777777777
+ },
+ "sent": 3605070.980492208,
+ "first": 3605071.07916575,
+ "last": 3605072.701888375,
+ "chunks": 145
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 255.09658594530208,
+ "ttft_max": 0.4042953331954777,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 69.04753394701699,
+ "seconds": 3.693108,
+ "prefill_seconds": 0.103401542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.403547042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605072.703041333,
+ "first": 3605073.107336666,
+ "last": 3605076.800417541,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 65.40786452620499,
+ "seconds": 3.89861375,
+ "prefill_seconds": 0.09968425,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.19807675,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605072.702960125,
+ "first": 3605072.901815625,
+ "last": 3605076.800355916,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 67.16709150848577,
+ "seconds": 3.796502041,
+ "prefill_seconds": 0.102110291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.300229167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605072.703119333,
+ "first": 3605073.003936041,
+ "last": 3605076.800587125,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 63.77710860351041,
+ "seconds": 3.998299791,
+ "prefill_seconds": 0.098301959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098602667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605072.702881125,
+ "first": 3605072.802101625,
+ "last": 3605076.800324375,
+ "chunks": 167
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 278.7918956411484,
+ "ttft_max": 0.8300348329357803,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.85159345841741,
+ "seconds": 7.316738625,
+ "prefill_seconds": 0.101190292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.101419667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605076.801830875,
+ "first": 3605076.904127708,
+ "last": 3605084.22079525,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 38.09187846980368,
+ "seconds": 6.694340375,
+ "prefill_seconds": 0.104884833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.723694375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605076.802039125,
+ "first": 3605077.526568416,
+ "last": 3605084.221311625,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 38.70093873857531,
+ "seconds": 6.588987459,
+ "prefill_seconds": 0.105350917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.829023,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605076.8018925,
+ "first": 3605077.631927333,
+ "last": 3605084.221138708,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.38489569109685,
+ "seconds": 7.008402667,
+ "prefill_seconds": 0.10326725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.409696084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605076.802100708,
+ "first": 3605077.212518,
+ "last": 3605084.220947666,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.85689134832941,
+ "seconds": 7.111603667,
+ "prefill_seconds": 0.102814666,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.306445875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605076.802176333,
+ "first": 3605077.109206125,
+ "last": 3605084.221413583,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.934623923240295,
+ "seconds": 6.904090875,
+ "prefill_seconds": 0.104301042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.513925792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605076.801991083,
+ "first": 3605077.316830916,
+ "last": 3605084.221197833,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.34592633607471,
+ "seconds": 7.214409875,
+ "prefill_seconds": 0.102321,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.203729291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605076.801947291,
+ "first": 3605077.006408833,
+ "last": 3605084.22084075,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 37.504499161057446,
+ "seconds": 6.79918425,
+ "prefill_seconds": 0.10486425,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.618898666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605076.801746666,
+ "first": 3605077.421678791,
+ "last": 3605084.220826541,
+ "chunks": 196
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 312.84454795576346,
+ "ttft_max": 1.64287120802328,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.680087870793056,
+ "seconds": 12.330701958,
+ "prefill_seconds": 0.103426583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.807823167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.223774208,
+ "first": 3605085.034211333,
+ "last": 3605097.365090666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.78272141148618,
+ "seconds": 11.70652625,
+ "prefill_seconds": 0.104171917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.432023125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.225127125,
+ "first": 3605085.658580458,
+ "last": 3605097.365670583,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.339825642826977,
+ "seconds": 12.536980625,
+ "prefill_seconds": 0.102136875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.6015465,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224564125,
+ "first": 3605084.827927208,
+ "last": 3605097.365589625,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 22.17720418879119,
+ "seconds": 11.498293375,
+ "prefill_seconds": 0.104055375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.639223542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.22396625,
+ "first": 3605085.866837458,
+ "last": 3605097.365761166,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.97827367921699,
+ "seconds": 11.602367125,
+ "prefill_seconds": 0.104170167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.5356085,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.223871666,
+ "first": 3605085.76279375,
+ "last": 3605097.365196416,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.508063343255248,
+ "seconds": 12.434133625,
+ "prefill_seconds": 0.102846541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.7043205,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224685291,
+ "first": 3605084.930803125,
+ "last": 3605097.365631,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.854728882925233,
+ "seconds": 12.227442583,
+ "prefill_seconds": 0.103323125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.911108458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224626083,
+ "first": 3605085.137543958,
+ "last": 3605097.365725875,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.401224171004174,
+ "seconds": 11.91520625,
+ "prefill_seconds": 0.10408875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.223428792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224805583,
+ "first": 3605085.449904833,
+ "last": 3605097.365363208,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.704330976395816,
+ "seconds": 12.941317333,
+ "prefill_seconds": 0.099322166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.197725875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224089458,
+ "first": 3605084.423510833,
+ "last": 3605097.36485925,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.85722196718048,
+ "seconds": 12.841675458,
+ "prefill_seconds": 0.099694667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.297431583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224746333,
+ "first": 3605084.523204583,
+ "last": 3605097.365503916,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.175482988453066,
+ "seconds": 12.639102625,
+ "prefill_seconds": 0.10217,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.499395083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.2242065,
+ "first": 3605084.725776375,
+ "last": 3605097.365019708,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.554265314859858,
+ "seconds": 13.040633125,
+ "prefill_seconds": 0.098349833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098607125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224469916,
+ "first": 3605084.32414025,
+ "last": 3605097.364759083,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.032647235315533,
+ "seconds": 12.124008792,
+ "prefill_seconds": 0.103512084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.014515417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224311875,
+ "first": 3605085.241059291,
+ "last": 3605097.365551041,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.01370291147283,
+ "seconds": 12.741270375,
+ "prefill_seconds": 0.10039225,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.397747709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.224878875,
+ "first": 3605084.623611958,
+ "last": 3605097.365279708,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.21588669643829,
+ "seconds": 12.019294958,
+ "prefill_seconds": 0.10470325,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.119084083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.22439125,
+ "first": 3605085.345772541,
+ "last": 3605097.3653205,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.59063720686525,
+ "seconds": 11.810675042,
+ "prefill_seconds": 0.104535833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.327713417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605084.223643833,
+ "first": 3605085.554434666,
+ "last": 3605097.365423625,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 135.59348465320016,
+ "ttft_max": 0.08575020777061582,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 135.5871612465971,
+ "seconds": 1.880709041,
+ "prefill_seconds": 0.085088958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.085327083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 204,
+ "drafted": 90,
+ "accepted": 51,
+ "acceptance_rate": 0.5666666666666667,
+ "tokens_per_round": 1.2549019607843137
+ },
+ "sent": 3605102.731765583,
+ "first": 3605102.817515791,
+ "last": 3605104.698137125,
+ "chunks": 205
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 225.62437397965493,
+ "ttft_max": 0.34385187504813075,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 58.62565036902639,
+ "seconds": 4.349631917,
+ "prefill_seconds": 0.085715541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.256372,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605104.700027083,
+ "first": 3605104.957033833,
+ "last": 3605109.306794875,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 57.49275017434879,
+ "seconds": 4.435341834,
+ "prefill_seconds": 0.085297834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.170642833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605104.700094708,
+ "first": 3605104.871325708,
+ "last": 3605109.306695375,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 59.82188035125414,
+ "seconds": 4.262654375,
+ "prefill_seconds": 0.086960625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.343317417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605104.700153458,
+ "first": 3605105.044005333,
+ "last": 3605109.306778791,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 56.40822558708206,
+ "seconds": 4.520617292,
+ "prefill_seconds": 0.085123083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.08533825,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605104.699931625,
+ "first": 3605104.786006708,
+ "last": 3605109.306667166,
+ "chunks": 216
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 239.06832963446098,
+ "ttft_max": 0.8974757078103721,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 29.885293065711632,
+ "seconds": 8.532625042,
+ "prefill_seconds": 0.109820125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.110008416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 39,
+ "accepted": 34,
+ "acceptance_rate": 0.8717948717948718,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605109.308800958,
+ "first": 3605109.419628375,
+ "last": 3605117.952543958,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 31.982160390065538,
+ "seconds": 7.973194959,
+ "prefill_seconds": 0.113419125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.66929175,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 39,
+ "accepted": 34,
+ "acceptance_rate": 0.8717948717948718,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605109.308735791,
+ "first": 3605109.979150333,
+ "last": 3605117.952352875,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 30.682415268592585,
+ "seconds": 8.310949375,
+ "prefill_seconds": 0.110996166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.331544792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 39,
+ "accepted": 34,
+ "acceptance_rate": 0.8717948717948718,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605109.309026541,
+ "first": 3605109.641333083,
+ "last": 3605117.952246875,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 31.533558569504233,
+ "seconds": 8.086623,
+ "prefill_seconds": 0.112515709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.555912916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 39,
+ "accepted": 34,
+ "acceptance_rate": 0.8717948717948718,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605109.308859125,
+ "first": 3605109.865747875,
+ "last": 3605117.952323583,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 31.100853907471972,
+ "seconds": 8.199131791,
+ "prefill_seconds": 0.111869375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.443369,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 39,
+ "accepted": 34,
+ "acceptance_rate": 0.8717948717948718,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605109.30925225,
+ "first": 3605109.753270708,
+ "last": 3605117.952603083,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.44146962529053,
+ "seconds": 7.86030975,
+ "prefill_seconds": 0.112904125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.782198375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 39,
+ "accepted": 34,
+ "acceptance_rate": 0.8717948717948718,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605109.308911541,
+ "first": 3605110.092090125,
+ "last": 3605117.952276583,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 30.278069487121737,
+ "seconds": 8.421937208,
+ "prefill_seconds": 0.110679667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.220623334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 39,
+ "accepted": 34,
+ "acceptance_rate": 0.8717948717948718,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605109.308964416,
+ "first": 3605109.530324958,
+ "last": 3605117.952205375,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.9190156055533,
+ "seconds": 7.746282667,
+ "prefill_seconds": 0.114026709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.896113292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 39,
+ "accepted": 34,
+ "acceptance_rate": 0.8717948717948718,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605109.308652208,
+ "first": 3605110.206127916,
+ "last": 3605117.952753625,
+ "chunks": 222
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 291.00522378471396,
+ "ttft_max": 1.6490971660241485,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.628776725782256,
+ "seconds": 12.9911305,
+ "prefill_seconds": 0.103253458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.129239958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.954696041,
+ "first": 3605119.086046041,
+ "last": 3605132.077600958,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.876988695385773,
+ "seconds": 13.5085105,
+ "prefill_seconds": 0.102846959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.61226975,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.955237666,
+ "first": 3605118.568501666,
+ "last": 3605132.077354166,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.02239297451555,
+ "seconds": 13.4052535,
+ "prefill_seconds": 0.103267709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.715408334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.955115041,
+ "first": 3605118.671778916,
+ "last": 3605132.077752458,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.473994241940837,
+ "seconds": 13.094386125,
+ "prefill_seconds": 0.103817167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.026012584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.955405916,
+ "first": 3605118.982773208,
+ "last": 3605132.077111625,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.455898082539104,
+ "seconds": 13.816721292,
+ "prefill_seconds": 0.10144075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.304203416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.954764791,
+ "first": 3605118.26023325,
+ "last": 3605132.076844166,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.32141160864201,
+ "seconds": 13.918141541,
+ "prefill_seconds": 0.101377208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.20284425,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.954831125,
+ "first": 3605118.158791583,
+ "last": 3605132.076891458,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.73436891317599,
+ "seconds": 13.611347208,
+ "prefill_seconds": 0.102797667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.50943175,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.955307625,
+ "first": 3605118.465649166,
+ "last": 3605132.077550958,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.59395354800949,
+ "seconds": 13.714135584,
+ "prefill_seconds": 0.102592667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.406669708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.955194083,
+ "first": 3605118.362816333,
+ "last": 3605132.077146791,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.188910288888422,
+ "seconds": 14.019531459,
+ "prefill_seconds": 0.101305167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.101564583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.954890958,
+ "first": 3605118.057434458,
+ "last": 3605132.076825458,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.947034626440626,
+ "seconds": 12.783855083,
+ "prefill_seconds": 0.103593084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.336430334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.954939458,
+ "first": 3605119.293258,
+ "last": 3605132.077039208,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.109461251593757,
+ "seconds": 12.680598292,
+ "prefill_seconds": 0.103316959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.439643042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.955498541,
+ "first": 3605119.39657675,
+ "last": 3605132.077682041,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.17055943357387,
+ "seconds": 13.30164625,
+ "prefill_seconds": 0.103561666,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.81862925,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.954983666,
+ "first": 3605118.775327708,
+ "last": 3605132.077278,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.320957071673337,
+ "seconds": 13.198103958,
+ "prefill_seconds": 0.103588,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.922272042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.95503025,
+ "first": 3605118.8789235,
+ "last": 3605132.077801333,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.27512247938112,
+ "seconds": 12.576989375,
+ "prefill_seconds": 0.103615666,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.543083667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.95506975,
+ "first": 3605119.500232166,
+ "last": 3605132.077231583,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.78663726380439,
+ "seconds": 12.887485458,
+ "prefill_seconds": 0.103617125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.232856375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.955829083,
+ "first": 3605119.189652875,
+ "last": 3605132.077437666,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.443263874905824,
+ "seconds": 12.473546375,
+ "prefill_seconds": 0.103488792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.64631775,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605117.95461375,
+ "first": 3605119.603710916,
+ "last": 3605132.077716833,
+ "chunks": 229
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 151.98627432059348,
+ "ttft_max": 0.10828037513419986,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 151.98426635034073,
+ "seconds": 1.67780525,
+ "prefill_seconds": 0.10773275,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.107901625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 144,
+ "drafted": 180,
+ "accepted": 111,
+ "acceptance_rate": 0.6166666666666667,
+ "tokens_per_round": 1.7777777777777777
+ },
+ "sent": 3605132.079570916,
+ "first": 3605132.187851291,
+ "last": 3605133.865634375,
+ "chunks": 145
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 232.97216394450493,
+ "ttft_max": 0.4620435410179198,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 58.24617511786785,
+ "seconds": 4.377969875,
+ "prefill_seconds": 0.112383917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.112573583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605133.868006958,
+ "first": 3605133.981091541,
+ "last": 3605138.359214,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 63.291191598297736,
+ "seconds": 4.028996667,
+ "prefill_seconds": 0.118243792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.461524042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605133.868063,
+ "first": 3605134.330106541,
+ "last": 3605138.359297,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 59.809433662600966,
+ "seconds": 4.263541458,
+ "prefill_seconds": 0.11442525,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.226997416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605133.868110208,
+ "first": 3605134.095524666,
+ "last": 3605138.359019625,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 61.48683856136229,
+ "seconds": 4.147229,
+ "prefill_seconds": 0.11631375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.343276458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605133.867932791,
+ "first": 3605134.2118565,
+ "last": 3605138.359159125,
+ "chunks": 167
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 273.27415594074364,
+ "ttft_max": 0.904660833068192,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.21827568262825,
+ "seconds": 7.240558916,
+ "prefill_seconds": 0.112328125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.335218041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605138.361518583,
+ "first": 3605138.697389458,
+ "last": 3605145.938332166,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.7781916277839,
+ "seconds": 7.12724675,
+ "prefill_seconds": 0.113313125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.448416083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605138.361304,
+ "first": 3605138.8107295,
+ "last": 3605145.93808775,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.35921233541174,
+ "seconds": 7.01335325,
+ "prefill_seconds": 0.113931333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.562334125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605138.361365958,
+ "first": 3605138.924684708,
+ "last": 3605145.9379155,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 37.58056725385068,
+ "seconds": 6.785421792,
+ "prefill_seconds": 0.11387375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.790247917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605138.361588166,
+ "first": 3605139.152628875,
+ "last": 3605145.937980125,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 38.22085397307286,
+ "seconds": 6.671750458,
+ "prefill_seconds": 0.113680292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.903884875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605138.3616655,
+ "first": 3605139.266326333,
+ "last": 3605145.938176333,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.16111921070131,
+ "seconds": 7.464626625,
+ "prefill_seconds": 0.110947917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.11120125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605138.361424166,
+ "first": 3605138.473301333,
+ "last": 3605145.937860666,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.68028146516437,
+ "seconds": 7.3528815,
+ "prefill_seconds": 0.111733208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.222910333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605138.361467666,
+ "first": 3605138.585063458,
+ "last": 3605145.938006166,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.960386954623175,
+ "seconds": 6.899278417,
+ "prefill_seconds": 0.1140825,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.67635775,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605138.361225083,
+ "first": 3605139.038731333,
+ "last": 3605145.93793625,
+ "chunks": 196
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 305.3379633728433,
+ "ttft_max": 1.633806750178337,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.822927297339806,
+ "seconds": 12.863892208,
+ "prefill_seconds": 0.101247709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.59563525,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.940995291,
+ "first": 3605146.538419875,
+ "last": 3605159.402413541,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.370879482298406,
+ "seconds": 11.93212475,
+ "prefill_seconds": 0.104167625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.526834291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941094541,
+ "first": 3605147.470343208,
+ "last": 3605159.402632,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.142649520347533,
+ "seconds": 12.659704958,
+ "prefill_seconds": 0.102535625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.799723584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941823625,
+ "first": 3605146.7426835,
+ "last": 3605159.402381083,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.477246794794308,
+ "seconds": 12.452845959,
+ "prefill_seconds": 0.103823583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.006469416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941530375,
+ "first": 3605146.949606708,
+ "last": 3605159.402607833,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.00382553794195,
+ "seconds": 12.1406455,
+ "prefill_seconds": 0.104434792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.318557541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941182916,
+ "first": 3605147.261812166,
+ "last": 3605159.402363333,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.668129306462152,
+ "seconds": 12.965137458,
+ "prefill_seconds": 0.100130583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.49446125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941742708,
+ "first": 3605146.437193875,
+ "last": 3605159.402312875,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.185936302711863,
+ "seconds": 12.036286542,
+ "prefill_seconds": 0.104347459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.422695625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.942140416,
+ "first": 3605147.366185,
+ "last": 3605159.402436958,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.517372686097072,
+ "seconds": 13.065283125,
+ "prefill_seconds": 0.099564083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.394375834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941583333,
+ "first": 3605146.337003916,
+ "last": 3605159.40296525,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.084778292035907,
+ "seconds": 13.361433709,
+ "prefill_seconds": 0.09827075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098502,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941264916,
+ "first": 3605146.040833125,
+ "last": 3605159.402510166,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.560151994421332,
+ "seconds": 11.827374875,
+ "prefill_seconds": 0.104764667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.6315635,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941330583,
+ "first": 3605147.575137333,
+ "last": 3605159.402668125,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.649004277684163,
+ "seconds": 12.34926375,
+ "prefill_seconds": 0.1036195,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.110109584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941380125,
+ "first": 3605147.053187208,
+ "last": 3605159.403075916,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.82467850167173,
+ "seconds": 12.245087,
+ "prefill_seconds": 0.104157708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.21415075,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.94190325,
+ "first": 3605147.157471791,
+ "last": 3605159.402822958,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.307930746690957,
+ "seconds": 12.55667075,
+ "prefill_seconds": 0.103049709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.902659542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941425291,
+ "first": 3605146.845705916,
+ "last": 3605159.402448333,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.225912789847122,
+ "seconds": 13.263349459,
+ "prefill_seconds": 0.098078833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.196348167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941657666,
+ "first": 3605146.138916416,
+ "last": 3605159.402203416,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.980784034931435,
+ "seconds": 12.762261959,
+ "prefill_seconds": 0.101674958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.697299833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.941471583,
+ "first": 3605146.640102875,
+ "last": 3605159.402835125,
+ "chunks": 203
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.36980007410188,
+ "seconds": 13.164823541,
+ "prefill_seconds": 0.0985185,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.294843917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 202,
+ "drafted": 53,
+ "accepted": 53,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.2673267326732673
+ },
+ "sent": 3605145.940892041,
+ "first": 3605146.237452041,
+ "last": 3605159.402982875,
+ "chunks": 203
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 124.69219413734717,
+ "ttft_max": 0.09309358289465308,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 124.68141557488158,
+ "seconds": 2.045212583,
+ "prefill_seconds": 0.092243959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.092521084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 86,
+ "drafted": 275,
+ "accepted": 169,
+ "acceptance_rate": 0.6145454545454545,
+ "tokens_per_round": 2.9767441860465116
+ },
+ "sent": 3605165.7339885,
+ "first": 3605165.827082083,
+ "last": 3605167.872117875,
+ "chunks": 87
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 208.12402438377157,
+ "ttft_max": 0.4569550408050418,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.27837233945698,
+ "seconds": 4.78618225,
+ "prefill_seconds": 0.114795834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.224114042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 218,
+ "drafted": 46,
+ "accepted": 37,
+ "acceptance_rate": 0.8043478260869565,
+ "tokens_per_round": 1.1743119266055047
+ },
+ "sent": 3605167.875097708,
+ "first": 3605168.099788708,
+ "last": 3605172.8859015,
+ "chunks": 219
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 55.99561521394278,
+ "seconds": 4.553928,
+ "prefill_seconds": 0.116606292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.456425666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 218,
+ "drafted": 46,
+ "accepted": 37,
+ "acceptance_rate": 0.8043478260869565,
+ "tokens_per_round": 1.1743119266055047
+ },
+ "sent": 3605167.875204875,
+ "first": 3605168.332159916,
+ "last": 3605172.885936333,
+ "chunks": 219
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.03033548440874,
+ "seconds": 4.900987042,
+ "prefill_seconds": 0.109144042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.109344458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 218,
+ "drafted": 46,
+ "accepted": 37,
+ "acceptance_rate": 0.8043478260869565,
+ "tokens_per_round": 1.1743119266055047
+ },
+ "sent": 3605167.875152166,
+ "first": 3605167.985012458,
+ "last": 3605172.885886541,
+ "chunks": 219
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.59856064422458,
+ "seconds": 4.670452792,
+ "prefill_seconds": 0.115727375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.339796125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 218,
+ "drafted": 46,
+ "accepted": 37,
+ "acceptance_rate": 0.8043478260869565,
+ "tokens_per_round": 1.1743119266055047
+ },
+ "sent": 3605167.875021041,
+ "first": 3605168.215677,
+ "last": 3605172.885920875,
+ "chunks": 219
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 251.6911281708875,
+ "ttft_max": 0.8647510409355164,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.220280315461096,
+ "seconds": 7.451721542,
+ "prefill_seconds": 0.109500625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.75442575,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605172.889201708,
+ "first": 3605173.644758291,
+ "last": 3605181.09655925,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.784678873897505,
+ "seconds": 7.778023417,
+ "prefill_seconds": 0.107079834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.428348125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605172.888983666,
+ "first": 3605173.318376625,
+ "last": 3605181.096337708,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.24804015841713,
+ "seconds": 7.669625,
+ "prefill_seconds": 0.108443959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.536531417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605172.889049125,
+ "first": 3605173.426806583,
+ "last": 3605181.096312666,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.729339277725074,
+ "seconds": 7.342495,
+ "prefill_seconds": 0.109349667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.863685958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605172.8893795,
+ "first": 3605173.754130541,
+ "last": 3605181.096878333,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.724893118953005,
+ "seconds": 7.561180375,
+ "prefill_seconds": 0.1084465,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.644938125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605172.889145291,
+ "first": 3605173.535275375,
+ "last": 3605181.09632675,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.33946614750428,
+ "seconds": 7.885102334,
+ "prefill_seconds": 0.106294208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.321225417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605172.889298375,
+ "first": 3605173.211260166,
+ "last": 3605181.096401833,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 31.909315309593254,
+ "seconds": 7.991396792,
+ "prefill_seconds": 0.11322875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.215124917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605172.889103083,
+ "first": 3605173.104958333,
+ "last": 3605181.096496208,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 31.4635540533525,
+ "seconds": 8.10461525,
+ "prefill_seconds": 0.1015975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.101866833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605172.888902375,
+ "first": 3605172.991705875,
+ "last": 3605181.096285333,
+ "chunks": 228
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 313.42934103469173,
+ "ttft_max": 1.5213572499342263,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.304617493868918,
+ "seconds": 11.96923625,
+ "prefill_seconds": 0.094936375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.136720792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.1012335,
+ "first": 3605182.239094333,
+ "last": 3605194.208778666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.18264648591888,
+ "seconds": 12.634616584,
+ "prefill_seconds": 0.094024083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.471642333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100237125,
+ "first": 3605181.573688583,
+ "last": 3605194.208247916,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.649907776155853,
+ "seconds": 11.778341166,
+ "prefill_seconds": 0.095603625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.327519875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100312666,
+ "first": 3605182.430033666,
+ "last": 3605194.208865708,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.88634233985453,
+ "seconds": 12.822870875,
+ "prefill_seconds": 0.093289083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.283379084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100461416,
+ "first": 3605181.385283833,
+ "last": 3605194.208107541,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.47560058642282,
+ "seconds": 11.873940334,
+ "prefill_seconds": 0.095297208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.231966833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100956208,
+ "first": 3605182.334405541,
+ "last": 3605194.208390541,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.33497099959012,
+ "seconds": 12.539973625,
+ "prefill_seconds": 0.094606625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.566118125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100694333,
+ "first": 3605181.668279125,
+ "last": 3605194.208471791,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.82723891494671,
+ "seconds": 11.682650334,
+ "prefill_seconds": 0.095696292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.423259709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100372083,
+ "first": 3605182.525757166,
+ "last": 3605194.208583583,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.742696023980194,
+ "seconds": 12.916169083,
+ "prefill_seconds": 0.100347583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.19014025,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100417375,
+ "first": 3605181.291986041,
+ "last": 3605194.208053541,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.033635098473024,
+ "seconds": 12.728593625,
+ "prefill_seconds": 0.094364917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.377678292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.101027916,
+ "first": 3605181.479648083,
+ "last": 3605194.208136083,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.971089533527618,
+ "seconds": 12.159597125,
+ "prefill_seconds": 0.095635834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.946418208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100824541,
+ "first": 3605182.048714083,
+ "last": 3605194.208658458,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.646486217524153,
+ "seconds": 12.350769875,
+ "prefill_seconds": 0.0944225,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.755228792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100615916,
+ "first": 3605181.857476375,
+ "last": 3605194.208165041,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.590524741306723,
+ "seconds": 13.016496667,
+ "prefill_seconds": 0.089569291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.089765958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.10051575,
+ "first": 3605181.191606166,
+ "last": 3605194.208012375,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.136996552358998,
+ "seconds": 12.064154875,
+ "prefill_seconds": 0.095451583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.04185975,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.101147708,
+ "first": 3605182.144160458,
+ "last": 3605194.208894166,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.489745150967604,
+ "seconds": 12.445249959,
+ "prefill_seconds": 0.094742375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.660845583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.1007485,
+ "first": 3605181.763080083,
+ "last": 3605194.20826575,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.80747677102762,
+ "seconds": 12.255210125,
+ "prefill_seconds": 0.095575875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.850826625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.10057275,
+ "first": 3605181.953056416,
+ "last": 3605194.208488833,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 22.00764844769439,
+ "seconds": 11.586880834,
+ "prefill_seconds": 0.095791084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.518658584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605181.100157833,
+ "first": 3605182.621515083,
+ "last": 3605194.208810041,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 144.5383970515986,
+ "ttft_max": 0.09813770884647965,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 144.53195240137714,
+ "seconds": 1.76431575,
+ "prefill_seconds": 0.097527583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.097719208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 78,
+ "drafted": 285,
+ "accepted": 177,
+ "acceptance_rate": 0.6210526315789474,
+ "tokens_per_round": 3.282051282051282
+ },
+ "sent": 3605194.211645666,
+ "first": 3605194.309783375,
+ "last": 3605196.074020458,
+ "chunks": 79
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 224.85815545136234,
+ "ttft_max": 0.46384895825758576,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 56.21441182975977,
+ "seconds": 4.536203292,
+ "prefill_seconds": 0.109698167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.109858792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 97,
+ "accepted": 89,
+ "acceptance_rate": 0.9175257731958762,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605196.076701458,
+ "first": 3605196.18695575,
+ "last": 3605200.72311825,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 60.964521487528344,
+ "seconds": 4.182760625,
+ "prefill_seconds": 0.119314042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.463325,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 97,
+ "accepted": 89,
+ "acceptance_rate": 0.9175257731958762,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605196.076561208,
+ "first": 3605196.540410166,
+ "last": 3605200.723148791,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 59.27368123032637,
+ "seconds": 4.302078,
+ "prefill_seconds": 0.118036208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.344047292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 97,
+ "accepted": 89,
+ "acceptance_rate": 0.9175257731958762,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605196.076630166,
+ "first": 3605196.421134458,
+ "last": 3605200.723136666,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 57.690874674474806,
+ "seconds": 4.420109791,
+ "prefill_seconds": 0.116078292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.226017625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 97,
+ "accepted": 89,
+ "acceptance_rate": 0.9175257731958762,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605196.076477958,
+ "first": 3605196.30318375,
+ "last": 3605200.723097416,
+ "chunks": 167
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 228.3324076029578,
+ "ttft_max": 1.0712138330563903,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 29.879040126457436,
+ "seconds": 8.534410708,
+ "prefill_seconds": 0.13294175,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.52166925,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 65,
+ "accepted": 60,
+ "acceptance_rate": 0.9230769230769231,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605200.726163041,
+ "first": 3605201.248800291,
+ "last": 3605209.783159708,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 28.54197716851694,
+ "seconds": 8.934209375,
+ "prefill_seconds": 0.121683084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.121890709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 65,
+ "accepted": 60,
+ "acceptance_rate": 0.9230769230769231,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605200.726410625,
+ "first": 3605200.848901708,
+ "last": 3605209.783069583,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 30.860278678874316,
+ "seconds": 8.263049166,
+ "prefill_seconds": 0.136530834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.793128791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 65,
+ "accepted": 60,
+ "acceptance_rate": 0.9230769230769231,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605200.72622,
+ "first": 3605201.520281958,
+ "last": 3605209.783206916,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 31.932770373201418,
+ "seconds": 7.985527,
+ "prefill_seconds": 0.1394635,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.070516375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 65,
+ "accepted": 60,
+ "acceptance_rate": 0.9230769230769231,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605200.726513375,
+ "first": 3605201.797727208,
+ "last": 3605209.7831435,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 31.384495157857014,
+ "seconds": 8.125031125,
+ "prefill_seconds": 0.138017042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.931052709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 65,
+ "accepted": 60,
+ "acceptance_rate": 0.9230769230769231,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605200.726452208,
+ "first": 3605201.658244208,
+ "last": 3605209.7832445,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 28.98285892631777,
+ "seconds": 8.798303875,
+ "prefill_seconds": 0.13589625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.257799375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 65,
+ "accepted": 60,
+ "acceptance_rate": 0.9230769230769231,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605200.726282916,
+ "first": 3605200.984814458,
+ "last": 3605209.783097333,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 29.42073754897265,
+ "seconds": 8.667355792,
+ "prefill_seconds": 0.131006333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.38884425,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 65,
+ "accepted": 60,
+ "acceptance_rate": 0.9230769230769231,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605200.726354375,
+ "first": 3605201.115897041,
+ "last": 3605209.783179041,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 30.358845572752436,
+ "seconds": 8.399528875,
+ "prefill_seconds": 0.134879375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.656546625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 65,
+ "accepted": 60,
+ "acceptance_rate": 0.9230769230769231,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605200.72609275,
+ "first": 3605201.383693708,
+ "last": 3605209.783224708,
+ "chunks": 196
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 282.3386996870639,
+ "ttft_max": 1.8823581668548286,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.226622070138276,
+ "seconds": 13.262860167,
+ "prefill_seconds": 0.119095916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.296116708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787171458,
+ "first": 3605211.085496583,
+ "last": 3605224.34824125,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.79779946245813,
+ "seconds": 14.327613958,
+ "prefill_seconds": 0.122848958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.231967375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787227958,
+ "first": 3605210.020624291,
+ "last": 3605224.34815675,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.055507778747845,
+ "seconds": 13.38195775,
+ "prefill_seconds": 0.119004625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.177078458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.788033583,
+ "first": 3605210.966369083,
+ "last": 3605224.348362791,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.5742003097195,
+ "seconds": 13.027352125,
+ "prefill_seconds": 0.11742125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.531489708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787339583,
+ "first": 3605211.321101208,
+ "last": 3605224.348435458,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.24437998196221,
+ "seconds": 13.976906875,
+ "prefill_seconds": 0.118000417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.582595542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787291625,
+ "first": 3605210.371332125,
+ "last": 3605224.348315875,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.399385883911336,
+ "seconds": 13.144746,
+ "prefill_seconds": 0.118168417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.414581041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787738208,
+ "first": 3605211.203957833,
+ "last": 3605224.3483335,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.400278522490467,
+ "seconds": 13.858485875,
+ "prefill_seconds": 0.118439167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.700956625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787458583,
+ "first": 3605210.489796333,
+ "last": 3605224.348448625,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.942747804926768,
+ "seconds": 14.211870042,
+ "prefill_seconds": 0.1157755,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.347734791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787552291,
+ "first": 3605210.136408458,
+ "last": 3605224.348429083,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.11232199448928,
+ "seconds": 12.678794625,
+ "prefill_seconds": 0.115700042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.880046584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787384666,
+ "first": 3605211.669742833,
+ "last": 3605224.34841975,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.93046255050799,
+ "seconds": 12.794484792,
+ "prefill_seconds": 0.116433,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.764330792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787492458,
+ "first": 3605211.554018,
+ "last": 3605224.348388208,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.722089689626134,
+ "seconds": 13.620274458,
+ "prefill_seconds": 0.119076291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.939076041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787769583,
+ "first": 3605210.728119333,
+ "last": 3605224.348404166,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.750736349295554,
+ "seconds": 12.910911041,
+ "prefill_seconds": 0.116433375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.647919833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787423583,
+ "first": 3605211.437522208,
+ "last": 3605224.3483775,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.091644716210116,
+ "seconds": 14.09490425,
+ "prefill_seconds": 0.116940458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.464571833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787609208,
+ "first": 3605210.253391583,
+ "last": 3605224.348343875,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.559834143429526,
+ "seconds": 13.739346916,
+ "prefill_seconds": 0.119137375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.819974375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787643166,
+ "first": 3605210.608930375,
+ "last": 3605224.34844175,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.646478088315806,
+ "seconds": 14.450475541,
+ "prefill_seconds": 0.109028625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.109224,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787675125,
+ "first": 3605209.89771975,
+ "last": 3605224.348114458,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.88748538643221,
+ "seconds": 13.501003166,
+ "prefill_seconds": 0.119320625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.058378042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605209.787071333,
+ "first": 3605210.847375166,
+ "last": 3605224.348277583,
+ "chunks": 254
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 135.3705299959295,
+ "ttft_max": 0.08670658292248845,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 135.36170933161313,
+ "seconds": 1.883841459,
+ "prefill_seconds": 0.085781333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.086096792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 204,
+ "drafted": 90,
+ "accepted": 51,
+ "acceptance_rate": 0.5666666666666667,
+ "tokens_per_round": 1.2549019607843137
+ },
+ "sent": 3605230.084579083,
+ "first": 3605230.171285666,
+ "last": 3605232.055004375,
+ "chunks": 205
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 247.18854085067397,
+ "ttft_max": 0.3422554158605635,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 64.4663127278345,
+ "seconds": 3.955554292,
+ "prefill_seconds": 0.085400125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.255194791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605232.05815025,
+ "first": 3605232.314051291,
+ "last": 3605236.269604041,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 63.10377137843549,
+ "seconds": 4.040962916,
+ "prefill_seconds": 0.085076667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169815167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605232.0582255,
+ "first": 3605232.228662958,
+ "last": 3605236.269935791,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 61.80205211183789,
+ "seconds": 4.126076583,
+ "prefill_seconds": 0.084549458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.084750125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605232.058285291,
+ "first": 3605232.143530916,
+ "last": 3605236.269508,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 65.90286679271894,
+ "seconds": 3.869330917,
+ "prefill_seconds": 0.086262041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.341408667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605232.058066125,
+ "first": 3605232.400321541,
+ "last": 3605236.269820875,
+ "chunks": 216
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 268.44417944890495,
+ "ttft_max": 0.7584767080843449,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.30311403855571,
+ "seconds": 7.223158833,
+ "prefill_seconds": 0.095596542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.468107459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605236.273302125,
+ "first": 3605236.74224275,
+ "last": 3605243.965309791,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.268515877396204,
+ "seconds": 7.0308915,
+ "prefill_seconds": 0.09632875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.66032425,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605236.273358083,
+ "first": 3605236.9345115,
+ "last": 3605243.965373625,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.97195388726913,
+ "seconds": 7.50619175,
+ "prefill_seconds": 0.093056167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.185104417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605236.27363875,
+ "first": 3605236.459160666,
+ "last": 3605243.965230541,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.39914261753111,
+ "seconds": 7.412975458,
+ "prefill_seconds": 0.093215666,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.278315125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605236.27341825,
+ "first": 3605236.552370416,
+ "last": 3605243.965326041,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.842132802854294,
+ "seconds": 7.318725333,
+ "prefill_seconds": 0.09426725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.372486833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605236.27355425,
+ "first": 3605236.646647666,
+ "last": 3605243.965269583,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.55590475435463,
+ "seconds": 7.599258666,
+ "prefill_seconds": 0.091830958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.092044417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605236.273475458,
+ "first": 3605236.366028375,
+ "last": 3605243.965252666,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.77833657849277,
+ "seconds": 7.127217875,
+ "prefill_seconds": 0.095941375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.563977417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605236.273594833,
+ "first": 3605236.83817375,
+ "last": 3605243.965293083,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.776712264682,
+ "seconds": 6.933735625,
+ "prefill_seconds": 0.097161791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.757486292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605236.273225458,
+ "first": 3605237.031702166,
+ "last": 3605243.965334,
+ "chunks": 229
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 319.2847676851127,
+ "ttft_max": 1.4483123333193362,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.80600240787437,
+ "seconds": 11.694027875,
+ "prefill_seconds": 0.091334042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.173616291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.969443166,
+ "first": 3605245.145292416,
+ "last": 3605256.839270791,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 22.150983325961008,
+ "seconds": 11.511904291,
+ "prefill_seconds": 0.091012292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.355748375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.96950325,
+ "first": 3605245.327496291,
+ "last": 3605256.839313541,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.145950469632904,
+ "seconds": 12.0590465,
+ "prefill_seconds": 0.090774708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.808576667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.970349458,
+ "first": 3605244.780217958,
+ "last": 3605256.839328166,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.2376016783258,
+ "seconds": 12.600307292,
+ "prefill_seconds": 0.089279583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.267583958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.969966791,
+ "first": 3605244.238831666,
+ "last": 3605256.839117791,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.833228656571702,
+ "seconds": 12.2400615,
+ "prefill_seconds": 0.090448792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.627604958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.970547625,
+ "first": 3605244.599190125,
+ "last": 3605256.839249541,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.98798123254718,
+ "seconds": 12.14981075,
+ "prefill_seconds": 0.090244583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.717820292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.970049083,
+ "first": 3605244.689422625,
+ "last": 3605256.83936075,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.68041589249006,
+ "seconds": 12.330506375,
+ "prefill_seconds": 0.09017325,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.537172167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.969576333,
+ "first": 3605244.508701625,
+ "last": 3605256.839198958,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.955637312919926,
+ "seconds": 12.778344084,
+ "prefill_seconds": 0.089494417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.089734333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.969684041,
+ "first": 3605244.060805875,
+ "last": 3605256.839061666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.095216256935384,
+ "seconds": 12.68958725,
+ "prefill_seconds": 0.088746,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.178434625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.96963825,
+ "first": 3605244.149560791,
+ "last": 3605256.8390755,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.637063857646872,
+ "seconds": 11.785332875,
+ "prefill_seconds": 0.091055709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.08228675,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.969914375,
+ "first": 3605245.053956541,
+ "last": 3605256.839345041,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 22.32804931103493,
+ "seconds": 11.420612542,
+ "prefill_seconds": 0.091313042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.446868375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.97046325,
+ "first": 3605245.418775583,
+ "last": 3605256.83936725,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.383060046456592,
+ "seconds": 12.5103885,
+ "prefill_seconds": 0.089931041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.357453166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.969826875,
+ "first": 3605244.32876,
+ "last": 3605256.839146,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.9772229504082,
+ "seconds": 11.602921833,
+ "prefill_seconds": 0.09111725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.264695667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.97027275,
+ "first": 3605245.236458291,
+ "last": 3605256.839352625,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.530285809656654,
+ "seconds": 12.420674625,
+ "prefill_seconds": 0.089750042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.447048958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.970631333,
+ "first": 3605244.41854875,
+ "last": 3605256.83918125,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.471178787040312,
+ "seconds": 11.876385667,
+ "prefill_seconds": 0.091522541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.99120025,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.969740833,
+ "first": 3605244.962909166,
+ "last": 3605256.8392915,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.307010309878976,
+ "seconds": 11.967892083,
+ "prefill_seconds": 0.091147875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.899696625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605243.969366083,
+ "first": 3605244.871349166,
+ "last": 3605256.839337791,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 157.47416781759355,
+ "ttft_max": 0.0967360413633287,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 157.4648286095686,
+ "seconds": 1.61940925,
+ "prefill_seconds": 0.096069542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.096265166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 144,
+ "drafted": 180,
+ "accepted": 111,
+ "acceptance_rate": 0.6166666666666667,
+ "tokens_per_round": 1.7777777777777777
+ },
+ "sent": 3605256.844516,
+ "first": 3605256.941252041,
+ "last": 3605258.56056525,
+ "chunks": 145
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 254.6708458908824,
+ "ttft_max": 0.3996269996277988,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 65.28964545138331,
+ "seconds": 3.905672917,
+ "prefill_seconds": 0.099539084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.19903375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605258.564169375,
+ "first": 3605258.763831125,
+ "last": 3605262.669454333,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 66.9946382614655,
+ "seconds": 3.80627475,
+ "prefill_seconds": 0.099390625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.298365042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605258.564251458,
+ "first": 3605258.863197083,
+ "last": 3605262.669443208,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 68.81561334793373,
+ "seconds": 3.705554417,
+ "prefill_seconds": 0.100711833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.399111875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605258.564329916,
+ "first": 3605258.963956916,
+ "last": 3605262.669422791,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 63.66673319241069,
+ "seconds": 4.005231417,
+ "prefill_seconds": 0.099261291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.09953825,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605258.564030833,
+ "first": 3605258.664284458,
+ "last": 3605262.669395041,
+ "chunks": 167
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 285.70738648598456,
+ "ttft_max": 0.8014092496596277,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.716172818794945,
+ "seconds": 7.13962275,
+ "prefill_seconds": 0.098861958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0990905,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605262.67330575,
+ "first": 3605262.773175375,
+ "last": 3605269.912740333,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 38.99465037162661,
+ "seconds": 6.539358542,
+ "prefill_seconds": 0.10073975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.699211458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605262.673546166,
+ "first": 3605263.37350375,
+ "last": 3605269.912882291,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 37.82942755620118,
+ "seconds": 6.740784,
+ "prefill_seconds": 0.100308,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.497804958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605262.673493833,
+ "first": 3605263.172063958,
+ "last": 3605269.91280275,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 37.274715233487235,
+ "seconds": 6.841098541,
+ "prefill_seconds": 0.099807333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.397588125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605262.673645791,
+ "first": 3605263.07171825,
+ "last": 3605269.9129505,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.73872080653342,
+ "seconds": 6.940905791,
+ "prefill_seconds": 0.099730791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.297757625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605262.673439583,
+ "first": 3605262.971927291,
+ "last": 3605269.912967,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.21838064353387,
+ "seconds": 7.040624,
+ "prefill_seconds": 0.098990417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.198081833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605262.673371833,
+ "first": 3605262.872175791,
+ "last": 3605269.913055333,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 39.61007249398499,
+ "seconds": 6.437756458,
+ "prefill_seconds": 0.101606083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.800571709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605262.673717208,
+ "first": 3605263.475126458,
+ "last": 3605269.913224208,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 38.403006695548854,
+ "seconds": 6.640105084,
+ "prefill_seconds": 0.100696167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.598439417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605262.673213166,
+ "first": 3605263.272753958,
+ "last": 3605269.913347791,
+ "chunks": 196
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 303.7043595048409,
+ "ttft_max": 1.6232123747467995,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.707553939051333,
+ "seconds": 12.939200917,
+ "prefill_seconds": 0.100283042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.591923375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917221625,
+ "first": 3605270.510649541,
+ "last": 3605283.449745416,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.1210335801187,
+ "seconds": 13.336099167,
+ "prefill_seconds": 0.097765375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.195100708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917575666,
+ "first": 3605270.113639666,
+ "last": 3605283.44979175,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.01970233116774,
+ "seconds": 12.737452125,
+ "prefill_seconds": 0.101138208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.793557834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917355333,
+ "first": 3605270.712412625,
+ "last": 3605283.449932916,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.346740438964275,
+ "seconds": 12.532719959,
+ "prefill_seconds": 0.103530875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.998317875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917632041,
+ "first": 3605270.917182083,
+ "last": 3605283.449850291,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.693519640614777,
+ "seconds": 12.322698334,
+ "prefill_seconds": 0.10384725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.208413917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.918093333,
+ "first": 3605271.127308625,
+ "last": 3605283.449959291,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.86200798449543,
+ "seconds": 12.838581084,
+ "prefill_seconds": 0.100614541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.692508416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917294666,
+ "first": 3605270.611324958,
+ "last": 3605283.449966333,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.26308732203688,
+ "seconds": 13.237753416,
+ "prefill_seconds": 0.09834925,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.293452084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917457625,
+ "first": 3605270.211989041,
+ "last": 3605283.449669833,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.180066868031723,
+ "seconds": 12.636231667,
+ "prefill_seconds": 0.101215833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.894750166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917404291,
+ "first": 3605270.813657875,
+ "last": 3605283.449834666,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.228878666832493,
+ "seconds": 12.011939208,
+ "prefill_seconds": 0.102896834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.518830125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917793791,
+ "first": 3605271.437999583,
+ "last": 3605283.44995075,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.412562914304186,
+ "seconds": 11.908896708,
+ "prefill_seconds": 0.103050708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.621631667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917860541,
+ "first": 3605271.541072916,
+ "last": 3605283.449972708,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.55593888604215,
+ "seconds": 13.039517125,
+ "prefill_seconds": 0.099267583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.491657291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917944833,
+ "first": 3605270.410338416,
+ "last": 3605283.449818,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.981866052192267,
+ "seconds": 13.433874167,
+ "prefill_seconds": 0.097258833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.097454292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917525875,
+ "first": 3605270.015855375,
+ "last": 3605283.449766875,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.87036352214025,
+ "seconds": 12.218282625,
+ "prefill_seconds": 0.104461041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.312728458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917714166,
+ "first": 3605271.231622583,
+ "last": 3605283.449863208,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.520543693562686,
+ "seconds": 12.426571333,
+ "prefill_seconds": 0.106122416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.104378167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917754708,
+ "first": 3605271.023299541,
+ "last": 3605283.449915125,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.048586514158238,
+ "seconds": 12.11482775,
+ "prefill_seconds": 0.103449667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.416263459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917914916,
+ "first": 3605271.335092041,
+ "last": 3605283.449897208,
+ "chunks": 220
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.408236402670443,
+ "seconds": 13.13875175,
+ "prefill_seconds": 0.099055875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.392426458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 219,
+ "drafted": 36,
+ "accepted": 36,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.1689497716894977
+ },
+ "sent": 3605269.917140958,
+ "first": 3605270.311076083,
+ "last": 3605283.449698625,
+ "chunks": 220
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 126.78640117388437,
+ "ttft_max": 0.09193537477403879,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 126.76524222550795,
+ "seconds": 2.011592417,
+ "prefill_seconds": 0.090170916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.090489583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 85,
+ "drafted": 283,
+ "accepted": 170,
+ "acceptance_rate": 0.6007067137809188,
+ "tokens_per_round": 3.011764705882353
+ },
+ "sent": 3605290.036222666,
+ "first": 3605290.128158041,
+ "last": 3605292.13941475,
+ "chunks": 86
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 200.43096587895258,
+ "ttft_max": 0.4380846251733601,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.650528654504505,
+ "seconds": 5.034498292,
+ "prefill_seconds": 0.104293291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.104680292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 197,
+ "drafted": 73,
+ "accepted": 58,
+ "acceptance_rate": 0.7945205479452054,
+ "tokens_per_round": 1.299492385786802
+ },
+ "sent": 3605292.143276458,
+ "first": 3605292.248608291,
+ "last": 3605297.2830055,
+ "chunks": 198
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.474920914161814,
+ "seconds": 4.859464208,
+ "prefill_seconds": 0.111324375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.325527875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 200,
+ "drafted": 74,
+ "accepted": 55,
+ "acceptance_rate": 0.7432432432432432,
+ "tokens_per_round": 1.28
+ },
+ "sent": 3605292.143338916,
+ "first": 3605292.469470583,
+ "last": 3605297.328782541,
+ "chunks": 201
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.61383015029026,
+ "seconds": 4.756235458,
+ "prefill_seconds": 0.112080791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.43766275,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 201,
+ "drafted": 69,
+ "accepted": 54,
+ "acceptance_rate": 0.782608695652174,
+ "tokens_per_round": 1.2736318407960199
+ },
+ "sent": 3605292.143407208,
+ "first": 3605292.581491833,
+ "last": 3605297.337642291,
+ "chunks": 202
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.42472493865987,
+ "seconds": 4.958704209,
+ "prefill_seconds": 0.109472208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.214242583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 199,
+ "drafted": 71,
+ "accepted": 56,
+ "acceptance_rate": 0.7887323943661971,
+ "tokens_per_round": 1.2864321608040201
+ },
+ "sent": 3605292.143192375,
+ "first": 3605292.358079333,
+ "last": 3605297.316705916,
+ "chunks": 200
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 255.46984362436277,
+ "ttft_max": 0.9172884998843074,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.96757134858678,
+ "seconds": 7.292471,
+ "prefill_seconds": 0.115811167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.800858792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 32,
+ "accepted": 28,
+ "acceptance_rate": 0.875,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605297.341384333,
+ "first": 3605298.143214708,
+ "last": 3605305.435554958,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.53055484152029,
+ "seconds": 7.176921417,
+ "prefill_seconds": 0.115590667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.91638425,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 32,
+ "accepted": 28,
+ "acceptance_rate": 0.875,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605297.341507458,
+ "first": 3605298.258795958,
+ "last": 3605305.435580625,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.38492431371953,
+ "seconds": 7.638178167,
+ "prefill_seconds": 0.114445042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.455078458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 32,
+ "accepted": 28,
+ "acceptance_rate": 0.875,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605297.341628541,
+ "first": 3605297.797422375,
+ "last": 3605305.435521375,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 31.934024719695735,
+ "seconds": 7.985213334,
+ "prefill_seconds": 0.107942584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.108172083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 32,
+ "accepted": 28,
+ "acceptance_rate": 0.875,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605297.341680875,
+ "first": 3605297.450301958,
+ "last": 3605305.435538041,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.42087377624303,
+ "seconds": 7.408295375,
+ "prefill_seconds": 0.11484875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.685039542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 32,
+ "accepted": 28,
+ "acceptance_rate": 0.875,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605297.341740041,
+ "first": 3605298.027373458,
+ "last": 3605305.43556775,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.895674014438306,
+ "seconds": 7.523083916,
+ "prefill_seconds": 0.115094834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.570190875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 32,
+ "accepted": 28,
+ "acceptance_rate": 0.875,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605297.341450916,
+ "first": 3605297.912514875,
+ "last": 3605305.435588875,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.892222827396125,
+ "seconds": 7.7525925,
+ "prefill_seconds": 0.113420959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.340682666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 32,
+ "accepted": 28,
+ "acceptance_rate": 0.875,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605297.341554,
+ "first": 3605297.682979916,
+ "last": 3605305.435493916,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.41796563335079,
+ "seconds": 7.866008709,
+ "prefill_seconds": 0.119201834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.227344958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 32,
+ "accepted": 28,
+ "acceptance_rate": 0.875,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605297.34129825,
+ "first": 3605297.569545375,
+ "last": 3605305.435476666,
+ "chunks": 228
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 313.8774581034392,
+ "ttft_max": 1.5110153751447797,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.060272004503304,
+ "seconds": 12.711692042,
+ "prefill_seconds": 0.093187083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.374716458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.44135625,
+ "first": 3605305.817483791,
+ "last": 3605318.530290208,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.67025183008689,
+ "seconds": 12.336569583,
+ "prefill_seconds": 0.094140792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.749637917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441405125,
+ "first": 3605306.192615291,
+ "last": 3605318.529260125,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.513701085871652,
+ "seconds": 12.430716375,
+ "prefill_seconds": 0.094110208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.655543625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.442013791,
+ "first": 3605306.098466166,
+ "last": 3605318.529441541,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.829214784946384,
+ "seconds": 12.242420208,
+ "prefill_seconds": 0.094183209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.843790334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441461916,
+ "first": 3605306.2868475,
+ "last": 3605318.530216875,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.359571456579687,
+ "seconds": 12.524821583,
+ "prefill_seconds": 0.093567333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.561439209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441520416,
+ "first": 3605306.004342125,
+ "last": 3605318.530478416,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.495463716658133,
+ "seconds": 11.862968083,
+ "prefill_seconds": 0.095485417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.223262791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441578041,
+ "first": 3605306.666461125,
+ "last": 3605318.529598458,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 22.02659359839882,
+ "seconds": 11.576914917,
+ "prefill_seconds": 0.095079667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.509074917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441627291,
+ "first": 3605306.952642666,
+ "last": 3605318.529668291,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.156408017755954,
+ "seconds": 12.05308575,
+ "prefill_seconds": 0.094809959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.033162541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441862375,
+ "first": 3605306.476218875,
+ "last": 3605318.52974675,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.99136372992379,
+ "seconds": 12.147852959,
+ "prefill_seconds": 0.094577583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.938381959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.44167525,
+ "first": 3605306.381410791,
+ "last": 3605318.529809208,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.669564145303198,
+ "seconds": 11.767657083,
+ "prefill_seconds": 0.095331208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.318576959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441896666,
+ "first": 3605306.76181175,
+ "last": 3605318.529422416,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.619602263921294,
+ "seconds": 12.997205375,
+ "prefill_seconds": 0.088972792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.089189584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.44171625,
+ "first": 3605305.531774958,
+ "last": 3605318.528930666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.847292069824867,
+ "seconds": 11.671927083,
+ "prefill_seconds": 0.095725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.414164333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441935625,
+ "first": 3605306.857545041,
+ "last": 3605318.529885,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.32386163636074,
+ "seconds": 11.958434375,
+ "prefill_seconds": 0.094735083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.127798667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441755666,
+ "first": 3605306.57101725,
+ "last": 3605318.530401416,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.914469759301284,
+ "seconds": 12.804759709,
+ "prefill_seconds": 0.092802833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.281539584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441971875,
+ "first": 3605305.724298875,
+ "last": 3605318.529047875,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.208637605857678,
+ "seconds": 12.618366709,
+ "prefill_seconds": 0.093288458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.467903583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441798791,
+ "first": 3605305.910813125,
+ "last": 3605318.529313708,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.77116191646302,
+ "seconds": 12.897572792,
+ "prefill_seconds": 0.099672334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.188832667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605305.441279833,
+ "first": 3605305.63146675,
+ "last": 3605318.528985041,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 147.83629271145213,
+ "ttft_max": 0.096926209051162,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 147.82846866597168,
+ "seconds": 1.724972208,
+ "prefill_seconds": 0.096228583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.096440459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 78,
+ "drafted": 284,
+ "accepted": 177,
+ "acceptance_rate": 0.6232394366197183,
+ "tokens_per_round": 3.282051282051282
+ },
+ "sent": 3605318.535977041,
+ "first": 3605318.63290325,
+ "last": 3605320.357784166,
+ "chunks": 79
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 220.9503576822706,
+ "ttft_max": 0.4242655001580715,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 59.40146559139531,
+ "seconds": 4.292823375,
+ "prefill_seconds": 0.109193292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.423730833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 10,
+ "accepted": 3,
+ "acceptance_rate": 0.3,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 3605320.361777708,
+ "first": 3605320.786043208,
+ "last": 3605325.078767416,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 55.23800969974208,
+ "seconds": 4.616386459,
+ "prefill_seconds": 0.099986875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.100170916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 10,
+ "accepted": 3,
+ "acceptance_rate": 0.3,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 3605320.361634458,
+ "first": 3605320.462448,
+ "last": 3605325.078736916,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 57.92797484039848,
+ "seconds": 4.402018208,
+ "prefill_seconds": 0.108133708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.314518667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 10,
+ "accepted": 3,
+ "acceptance_rate": 0.3,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 3605320.361707958,
+ "first": 3605320.67684525,
+ "last": 3605325.078869583,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 56.53916666194564,
+ "seconds": 4.510147833,
+ "prefill_seconds": 0.106229667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.206412666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 10,
+ "accepted": 3,
+ "acceptance_rate": 0.3,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 3605320.361551875,
+ "first": 3605320.568685541,
+ "last": 3605325.0787525,
+ "chunks": 253
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 257.0465470267213,
+ "ttft_max": 0.8670476251281798,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.559741330359046,
+ "seconds": 7.171030791,
+ "prefill_seconds": 0.110246708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.865938416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 43,
+ "accepted": 39,
+ "acceptance_rate": 0.9069767441860465,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 3605325.082999916,
+ "first": 3605325.950047541,
+ "last": 3605333.120935583,
+ "chunks": 217
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.0487811674307,
+ "seconds": 7.715867,
+ "prefill_seconds": 0.107012416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.321258417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 43,
+ "accepted": 39,
+ "acceptance_rate": 0.9069767441860465,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 3605325.08327425,
+ "first": 3605325.405153541,
+ "last": 3605333.1209695,
+ "chunks": 217
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.0007043812591,
+ "seconds": 7.499844625,
+ "prefill_seconds": 0.108330958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.537242959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 43,
+ "accepted": 39,
+ "acceptance_rate": 0.9069767441860465,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 3605325.083354583,
+ "first": 3605325.621150041,
+ "last": 3605333.120979291,
+ "chunks": 217
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.59696730918858,
+ "seconds": 7.822813625,
+ "prefill_seconds": 0.113354958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.214276167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 43,
+ "accepted": 39,
+ "acceptance_rate": 0.9069767441860465,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 3605325.083070083,
+ "first": 3605325.298106916,
+ "last": 3605333.120946875,
+ "chunks": 217
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.02128968234008,
+ "seconds": 7.28128525,
+ "prefill_seconds": 0.109204875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.755724292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 43,
+ "accepted": 39,
+ "acceptance_rate": 0.9069767441860465,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 3605325.083523375,
+ "first": 3605325.83974725,
+ "last": 3605333.121043833,
+ "chunks": 217
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.51667722496765,
+ "seconds": 7.60815275,
+ "prefill_seconds": 0.107665042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.428864333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 43,
+ "accepted": 39,
+ "acceptance_rate": 0.9069767441860465,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 3605325.083146,
+ "first": 3605325.512829125,
+ "last": 3605333.120899958,
+ "chunks": 217
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.50375520787656,
+ "seconds": 7.390499917,
+ "prefill_seconds": 0.109379,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.6465175,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 43,
+ "accepted": 39,
+ "acceptance_rate": 0.9069767441860465,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 3605325.083463166,
+ "first": 3605325.730562708,
+ "last": 3605333.121033916,
+ "chunks": 217
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.13133897047426,
+ "seconds": 7.936177208,
+ "prefill_seconds": 0.100699916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.100950125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 43,
+ "accepted": 39,
+ "acceptance_rate": 0.9069767441860465,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 3605325.082914666,
+ "first": 3605325.184738041,
+ "last": 3605333.121023791,
+ "chunks": 217
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 263.9872023279576,
+ "ttft_max": 2.16144795762375,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 16.500083985330427,
+ "seconds": 15.454466791,
+ "prefill_seconds": 0.122913333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.1231215,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126206875,
+ "first": 3605333.25041925,
+ "last": 3605348.704820291,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.817667675727197,
+ "seconds": 13.551094875,
+ "prefill_seconds": 0.134893458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.025968417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126846,
+ "first": 3605335.154130541,
+ "last": 3605348.705543875,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 16.9412854102989,
+ "seconds": 15.051986542,
+ "prefill_seconds": 0.133306958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.525254,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126274166,
+ "first": 3605333.6529565,
+ "last": 3605348.704856916,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.577823638472832,
+ "seconds": 14.506915375,
+ "prefill_seconds": 0.136994667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.070291541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126759583,
+ "first": 3605334.198122,
+ "last": 3605348.704980125,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.00578384558076,
+ "seconds": 13.416968333,
+ "prefill_seconds": 0.13424025,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.160254292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126926333,
+ "first": 3605335.288374291,
+ "last": 3605348.705384041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.747479878980165,
+ "seconds": 14.368237166,
+ "prefill_seconds": 0.138685917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.208875292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126330583,
+ "first": 3605334.33682525,
+ "last": 3605348.705066833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.09438622618824,
+ "seconds": 14.917177875,
+ "prefill_seconds": 0.134858,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.660051667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126624666,
+ "first": 3605333.787864,
+ "last": 3605348.704940416,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.252081909394978,
+ "seconds": 14.780824792,
+ "prefill_seconds": 0.136416833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.796385917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126385708,
+ "first": 3605333.924240208,
+ "last": 3605348.7050225,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.41324414474544,
+ "seconds": 14.644026,
+ "prefill_seconds": 0.136864667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.933301083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126715541,
+ "first": 3605334.061117541,
+ "last": 3605348.705056041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.092909645749767,
+ "seconds": 14.093918833,
+ "prefill_seconds": 0.137012834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.483268125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126442833,
+ "first": 3605334.611230625,
+ "last": 3605348.705081875,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 16.792604524165665,
+ "seconds": 15.185256083,
+ "prefill_seconds": 0.131768125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.392004542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126482833,
+ "first": 3605333.519721708,
+ "last": 3605348.705042458,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.449357599261333,
+ "seconds": 13.821619459,
+ "prefill_seconds": 0.135470416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.755477959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126891791,
+ "first": 3605334.883549125,
+ "last": 3605348.705171125,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.270250536285104,
+ "seconds": 13.957115667,
+ "prefill_seconds": 0.13683125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.620021208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126673666,
+ "first": 3605334.748088833,
+ "last": 3605348.705274708,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 16.648131928168212,
+ "seconds": 15.317033833,
+ "prefill_seconds": 0.137435125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.260257041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126563791,
+ "first": 3605333.387854375,
+ "last": 3605348.704874083,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.91874193182425,
+ "seconds": 14.230909791,
+ "prefill_seconds": 0.137384084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.346198542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126526375,
+ "first": 3605334.474237,
+ "last": 3605348.705713916,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.632244499725477,
+ "seconds": 13.685951792,
+ "prefill_seconds": 0.135675834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.891081125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 5,
+ "accepted": 4,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605333.126121875,
+ "first": 3605335.019221583,
+ "last": 3605348.705492625,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 135.21683007811936,
+ "ttft_max": 0.0871472917497158,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 135.2024288303004,
+ "seconds": 1.886060792,
+ "prefill_seconds": 0.086245667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.086543083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 204,
+ "drafted": 90,
+ "accepted": 51,
+ "acceptance_rate": 0.5666666666666667,
+ "tokens_per_round": 1.2549019607843137
+ },
+ "sent": 3605354.644582916,
+ "first": 3605354.731730208,
+ "last": 3605356.617590125,
+ "chunks": 205
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 241.76743130633574,
+ "ttft_max": 0.35046541690826416,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 61.718282989438244,
+ "seconds": 4.131676833,
+ "prefill_seconds": 0.086965875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.17304075,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605356.622275875,
+ "first": 3605356.795996958,
+ "last": 3605360.927636875,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 63.05896439746511,
+ "seconds": 4.04383425,
+ "prefill_seconds": 0.087845292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.26081375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605356.622413541,
+ "first": 3605356.883842,
+ "last": 3605360.927687291,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 64.47625109491958,
+ "seconds": 3.954944583,
+ "prefill_seconds": 0.08896475,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.349774417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605356.622349708,
+ "first": 3605356.972815125,
+ "last": 3605360.927937083,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 60.44555424989418,
+ "seconds": 4.218672542,
+ "prefill_seconds": 0.085891709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.086074041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605356.622183791,
+ "first": 3605356.709006541,
+ "last": 3605360.927656833,
+ "chunks": 216
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 270.7456803820994,
+ "ttft_max": 0.7304916670545936,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.10999689684354,
+ "seconds": 7.262888708,
+ "prefill_seconds": 0.091176541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.361503833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605360.933122416,
+ "first": 3605361.295646916,
+ "last": 3605368.558554916,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.25366513350512,
+ "seconds": 7.444458834,
+ "prefill_seconds": 0.09023775,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.179989584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605360.933404833,
+ "first": 3605361.114011958,
+ "last": 3605368.558453083,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.55645373853483,
+ "seconds": 7.171693833,
+ "prefill_seconds": 0.091205792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.452702834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605360.933335958,
+ "first": 3605361.386909875,
+ "last": 3605368.558520791,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.843515292583646,
+ "seconds": 7.534678292,
+ "prefill_seconds": 0.089659083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.089863125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605360.933188916,
+ "first": 3605361.023834791,
+ "last": 3605368.558379916,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.49754156803731,
+ "seconds": 6.986771959,
+ "prefill_seconds": 0.092288,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.637563583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605360.933269291,
+ "first": 3605361.571819208,
+ "last": 3605368.558564291,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.9856298318258,
+ "seconds": 6.894569625,
+ "prefill_seconds": 0.09220275,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.729732584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605360.933553041,
+ "first": 3605361.664044708,
+ "last": 3605368.558581041,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.67458892637427,
+ "seconds": 7.354088625,
+ "prefill_seconds": 0.090460375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.270407417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605360.933460708,
+ "first": 3605361.204480708,
+ "last": 3605368.558538208,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.021696924128804,
+ "seconds": 7.079066834,
+ "prefill_seconds": 0.092644084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.545317208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605360.93303275,
+ "first": 3605361.479543083,
+ "last": 3605368.558492,
+ "chunks": 229
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 317.14545460069326,
+ "ttft_max": 1.5035453750751913,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.264583340348953,
+ "seconds": 12.583530375,
+ "prefill_seconds": 0.094196833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.373004334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.564750375,
+ "first": 3605368.939365208,
+ "last": 3605381.523284958,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.114033885301385,
+ "seconds": 12.677715542,
+ "prefill_seconds": 0.093041542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.278851875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565608166,
+ "first": 3605368.845154833,
+ "last": 3605381.523831416,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 22.08067244265804,
+ "seconds": 11.548561334,
+ "prefill_seconds": 0.093428917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.407884875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565175666,
+ "first": 3605369.975220458,
+ "last": 3605381.523954666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.581051025094403,
+ "seconds": 12.390037792,
+ "prefill_seconds": 0.100268333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.566598,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.564819791,
+ "first": 3605369.133161083,
+ "last": 3605381.523207875,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.221400404997944,
+ "seconds": 12.016172125,
+ "prefill_seconds": 0.093754625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.940406041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.56523525,
+ "first": 3605369.507174916,
+ "last": 3605381.523527625,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 22.26102090220382,
+ "seconds": 11.45500025,
+ "prefill_seconds": 0.093674042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.501510833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565353291,
+ "first": 3605370.068898666,
+ "last": 3605381.523995208,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.823640472034473,
+ "seconds": 12.863429417,
+ "prefill_seconds": 0.092897375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.093108458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.564889291,
+ "first": 3605368.659426416,
+ "last": 3605381.523334791,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.05599916202474,
+ "seconds": 12.110562792,
+ "prefill_seconds": 0.093667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.84692175,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565460083,
+ "first": 3605369.413415208,
+ "last": 3605381.524185916,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.967498850440553,
+ "seconds": 12.770753208,
+ "prefill_seconds": 0.092665459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.185828791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.564947375,
+ "first": 3605368.752121291,
+ "last": 3605381.522796333,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.387795624969122,
+ "seconds": 11.922687334,
+ "prefill_seconds": 0.093571708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.033904875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565401291,
+ "first": 3605369.6007535,
+ "last": 3605381.524110041,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.736696626974393,
+ "seconds": 12.297040584,
+ "prefill_seconds": 0.093499584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.660084125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565037166,
+ "first": 3605369.226532875,
+ "last": 3605381.523677875,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.894685794925884,
+ "seconds": 12.204060042,
+ "prefill_seconds": 0.093236083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.753312333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.564991208,
+ "first": 3605369.319736166,
+ "last": 3605381.523785666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.415943287029275,
+ "seconds": 12.490238458,
+ "prefill_seconds": 0.093332875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.466354083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565313833,
+ "first": 3605369.032799916,
+ "last": 3605381.523712625,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.556892300832946,
+ "seconds": 11.829163334,
+ "prefill_seconds": 0.093681209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.127521958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565076666,
+ "first": 3605369.694434791,
+ "last": 3605381.524069916,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.729100870895973,
+ "seconds": 11.73541425,
+ "prefill_seconds": 0.093859833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.221267625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.565120791,
+ "first": 3605369.78833975,
+ "last": 3605381.524150666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.903448234267138,
+ "seconds": 11.642002541,
+ "prefill_seconds": 0.09342325,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.314510833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605368.564657416,
+ "first": 3605369.881750375,
+ "last": 3605381.523902958,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 155.2756224781027,
+ "ttft_max": 0.09816666599363089,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 155.26933117741075,
+ "seconds": 1.642307583,
+ "prefill_seconds": 0.097623791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.097776958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 144,
+ "drafted": 180,
+ "accepted": 111,
+ "acceptance_rate": 0.6166666666666667,
+ "tokens_per_round": 1.7777777777777777
+ },
+ "sent": 3605381.528660625,
+ "first": 3605381.626827291,
+ "last": 3605383.269068333,
+ "chunks": 145
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 261.3078294249977,
+ "ttft_max": 0.3997202501632273,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 67.02797961517537,
+ "seconds": 3.804381416,
+ "prefill_seconds": 0.099036,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.1972735,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605383.273448833,
+ "first": 3605383.471414916,
+ "last": 3605387.275758833,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 68.83537927939683,
+ "seconds": 3.704490375,
+ "prefill_seconds": 0.09988475,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.297147083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605383.273581875,
+ "first": 3605383.571316625,
+ "last": 3605387.275774583,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 70.7808129870301,
+ "seconds": 3.60267125,
+ "prefill_seconds": 0.101852208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.398936292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605383.273515041,
+ "first": 3605383.673235291,
+ "last": 3605387.275792125,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 65.32720632146649,
+ "seconds": 3.903427291,
+ "prefill_seconds": 0.097979833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098211333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605383.273371041,
+ "first": 3605383.372349958,
+ "last": 3605387.275720166,
+ "chunks": 167
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 280.2334342413751,
+ "ttft_max": 0.8117147092707455,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 37.084801340102665,
+ "seconds": 6.876132291,
+ "prefill_seconds": 0.102005167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.501668542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605387.280753,
+ "first": 3605387.783466291,
+ "last": 3605394.659875291,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 38.228965469386054,
+ "seconds": 6.670334833,
+ "prefill_seconds": 0.102883166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.70730875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605387.281004958,
+ "first": 3605387.989273833,
+ "last": 3605394.659580416,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.54287143821144,
+ "seconds": 6.978105167,
+ "prefill_seconds": 0.101316292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.39971725,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605387.280939791,
+ "first": 3605387.681480041,
+ "last": 3605394.659770625,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 37.64820566621007,
+ "seconds": 6.773231167,
+ "prefill_seconds": 0.102894541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.604616459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605387.280877833,
+ "first": 3605387.886363625,
+ "last": 3605394.659825291,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 38.83195706222774,
+ "seconds": 6.566756334,
+ "prefill_seconds": 0.103611958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.81094475,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605387.281200166,
+ "first": 3605388.092914875,
+ "last": 3605394.659544541,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.516121262368586,
+ "seconds": 7.179838083,
+ "prefill_seconds": 0.09941125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.198144292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605387.280812791,
+ "first": 3605387.479661916,
+ "last": 3605394.659483416,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.01983336947377,
+ "seconds": 7.079433083,
+ "prefill_seconds": 0.100446917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.29840925,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605387.281063625,
+ "first": 3605387.580137125,
+ "last": 3605394.6596125,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.03117754607157,
+ "seconds": 7.279230042,
+ "prefill_seconds": 0.098551667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098811084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605387.280672125,
+ "first": 3605387.38023,
+ "last": 3605394.659435708,
+ "chunks": 196
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 278.70302133474246,
+ "ttft_max": 1.760789040941745,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.50306570402351,
+ "seconds": 13.7815,
+ "prefill_seconds": 0.110498333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.959904333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665737375,
+ "first": 3605395.626808416,
+ "last": 3605409.408287583,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.808313204366144,
+ "seconds": 13.557834625,
+ "prefill_seconds": 0.112213583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.183590458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665193875,
+ "first": 3605395.850536708,
+ "last": 3605409.40838575,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.96723081167307,
+ "seconds": 13.444239833,
+ "prefill_seconds": 0.113519625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.297130625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665262833,
+ "first": 3605395.964034166,
+ "last": 3605409.408241791,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.129585362089532,
+ "seconds": 13.330137333,
+ "prefill_seconds": 0.114137583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.411010958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.666038291,
+ "first": 3605396.078187125,
+ "last": 3605409.408339708,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.542866690280768,
+ "seconds": 14.535822708,
+ "prefill_seconds": 0.103120084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.205699625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665567166,
+ "first": 3605394.872240125,
+ "last": 3605409.408193541,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.46713693925592,
+ "seconds": 13.098998625,
+ "prefill_seconds": 0.11613475,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.642015666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.666172666,
+ "first": 3605396.309409916,
+ "last": 3605409.408378,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.801062083252255,
+ "seconds": 14.324987959,
+ "prefill_seconds": 0.105807542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.416487666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665355333,
+ "first": 3605395.083124083,
+ "last": 3605409.408067333,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.654002241477198,
+ "seconds": 13.669988708,
+ "prefill_seconds": 0.111501334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.071457708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.66611875,
+ "first": 3605395.738288291,
+ "last": 3605409.408216,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.93483167995468,
+ "seconds": 14.218142916,
+ "prefill_seconds": 0.1068725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.523246875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665833375,
+ "first": 3605395.189994625,
+ "last": 3605409.408132833,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.670543782090437,
+ "seconds": 14.430795291,
+ "prefill_seconds": 0.105029583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.31069625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665481,
+ "first": 3605394.977281125,
+ "last": 3605409.408325666,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.296077272876726,
+ "seconds": 13.215121208,
+ "prefill_seconds": 0.1150405,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.52606125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.666239916,
+ "first": 3605396.193254041,
+ "last": 3605409.408366625,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 17.419306216699344,
+ "seconds": 14.63892975,
+ "prefill_seconds": 0.102313917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.102579709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665622041,
+ "first": 3605394.76914725,
+ "last": 3605409.408009791,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.07181697629501,
+ "seconds": 14.110368666,
+ "prefill_seconds": 0.107833959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.631082958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.666305125,
+ "first": 3605395.297861458,
+ "last": 3605409.408104041,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.35592830677946,
+ "seconds": 13.891969708,
+ "prefill_seconds": 0.109609208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.849399459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665948833,
+ "first": 3605395.516287375,
+ "last": 3605409.408301083,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.643012259790854,
+ "seconds": 12.981715667,
+ "prefill_seconds": 0.1172645,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.759228,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.66590175,
+ "first": 3605396.426690791,
+ "last": 3605409.408348791,
+ "chunks": 222
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.212233857674743,
+ "seconds": 14.001577291,
+ "prefill_seconds": 0.108812375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.739878791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 221,
+ "drafted": 36,
+ "accepted": 34,
+ "acceptance_rate": 0.9444444444444444,
+ "tokens_per_round": 1.158371040723982
+ },
+ "sent": 3605394.665112375,
+ "first": 3605395.406704,
+ "last": 3605409.408167958,
+ "chunks": 222
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 134.84870970275804,
+ "ttft_max": 0.08884287532418966,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 134.8328003400667,
+ "seconds": 1.891231209,
+ "prefill_seconds": 0.088050667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.08823575,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 86,
+ "drafted": 274,
+ "accepted": 169,
+ "acceptance_rate": 0.6167883211678832,
+ "tokens_per_round": 2.9767441860465116
+ },
+ "sent": 3605415.381412625,
+ "first": 3605415.4702555,
+ "last": 3605417.361263583,
+ "chunks": 87
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 232.29052564817403,
+ "ttft_max": 0.39637737488374114,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 59.44672236125179,
+ "seconds": 4.28955525,
+ "prefill_seconds": 0.101351875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.195082083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 218,
+ "drafted": 48,
+ "accepted": 37,
+ "acceptance_rate": 0.7708333333333334,
+ "tokens_per_round": 1.1743119266055047
+ },
+ "sent": 3605417.367488291,
+ "first": 3605417.563336,
+ "last": 3605421.85298975,
+ "chunks": 219
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 62.36256925225751,
+ "seconds": 4.08899125,
+ "prefill_seconds": 0.100780375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.395675208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 218,
+ "drafted": 48,
+ "accepted": 37,
+ "acceptance_rate": 0.7708333333333334,
+ "tokens_per_round": 1.1743119266055047
+ },
+ "sent": 3605417.367552958,
+ "first": 3605417.763930333,
+ "last": 3605421.853057125,
+ "chunks": 219
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 60.86286961475636,
+ "seconds": 4.189746583,
+ "prefill_seconds": 0.0998135,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.294881542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 218,
+ "drafted": 48,
+ "accepted": 37,
+ "acceptance_rate": 0.7708333333333334,
+ "tokens_per_round": 1.1743119266055047
+ },
+ "sent": 3605417.367618916,
+ "first": 3605417.6631545,
+ "last": 3605421.853040291,
+ "chunks": 219
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 58.0744488389433,
+ "seconds": 4.390915542,
+ "prefill_seconds": 0.093563875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.093795125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 218,
+ "drafted": 48,
+ "accepted": 37,
+ "acceptance_rate": 0.7708333333333334,
+ "tokens_per_round": 1.1743119266055047
+ },
+ "sent": 3605417.367393208,
+ "first": 3605417.462004166,
+ "last": 3605421.852839708,
+ "chunks": 219
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 269.42968125685593,
+ "ttft_max": 0.7622620002366602,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.41528397005163,
+ "seconds": 7.002554208,
+ "prefill_seconds": 0.0957725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.665132083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605421.858775541,
+ "first": 3605422.524708875,
+ "last": 3605429.527125458,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.53646371845895,
+ "seconds": 7.383500583,
+ "prefill_seconds": 0.093928333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.284276834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605421.858546583,
+ "first": 3605422.143683916,
+ "last": 3605429.527139208,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.92196447130425,
+ "seconds": 6.906458084,
+ "prefill_seconds": 0.096173416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.761258875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605421.858615583,
+ "first": 3605422.620877583,
+ "last": 3605429.527359791,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.98216209825066,
+ "seconds": 7.289429375,
+ "prefill_seconds": 0.094067792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.378250666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605421.858727333,
+ "first": 3605422.237760875,
+ "last": 3605429.527248791,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.446830935158296,
+ "seconds": 7.193873,
+ "prefill_seconds": 0.09555475,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.473801166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605421.858674,
+ "first": 3605422.333324208,
+ "last": 3605429.527215875,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.10270598285169,
+ "seconds": 7.4774125,
+ "prefill_seconds": 0.093924667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.190319,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605421.858834375,
+ "first": 3605422.049752291,
+ "last": 3605429.527175708,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.92392179763727,
+ "seconds": 7.098334125,
+ "prefill_seconds": 0.095578625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.569345542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605421.859005666,
+ "first": 3605422.428939458,
+ "last": 3605429.527267541,
+ "chunks": 228
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.67961369351017,
+ "seconds": 7.57134575,
+ "prefill_seconds": 0.096190333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.096446583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 227,
+ "drafted": 31,
+ "accepted": 28,
+ "acceptance_rate": 0.9032258064516129,
+ "tokens_per_round": 1.1277533039647578
+ },
+ "sent": 3605421.858450958,
+ "first": 3605421.955810916,
+ "last": 3605429.527082541,
+ "chunks": 228
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 307.8591382227815,
+ "ttft_max": 1.5631438749842346,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.962722485692893,
+ "seconds": 12.773808792,
+ "prefill_seconds": 0.096862583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.567743333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.533959458,
+ "first": 3605430.103254916,
+ "last": 3605442.877523375,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.66563851479748,
+ "seconds": 12.966779584,
+ "prefill_seconds": 0.095515,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.3748605,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534009333,
+ "first": 3605429.910252666,
+ "last": 3605442.877322541,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.379971505472856,
+ "seconds": 13.157914083,
+ "prefill_seconds": 0.093905833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.183785542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534398958,
+ "first": 3605429.719103583,
+ "last": 3605442.876903291,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.10659876382601,
+ "seconds": 12.081529708,
+ "prefill_seconds": 0.100075125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.259563292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.5340755,
+ "first": 3605430.795726333,
+ "last": 3605442.877671583,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.28345692918465,
+ "seconds": 11.981136375,
+ "prefill_seconds": 0.1004525,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.359942458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534133125,
+ "first": 3605430.89622525,
+ "last": 3605442.877421458,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.933271174510256,
+ "seconds": 12.181564834,
+ "prefill_seconds": 0.099594042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.159546209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534439041,
+ "first": 3605430.695691541,
+ "last": 3605442.877127708,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.242652886598563,
+ "seconds": 13.251811042,
+ "prefill_seconds": 0.089549083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.089800042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534185125,
+ "first": 3605429.625141666,
+ "last": 3605442.876929125,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.52187078626089,
+ "seconds": 13.062272709,
+ "prefill_seconds": 0.095650833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.279312375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534242083,
+ "first": 3605429.814729958,
+ "last": 3605442.877101833,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.59748518639467,
+ "seconds": 12.380152125,
+ "prefill_seconds": 0.099545667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.961020708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534475958,
+ "first": 3605430.497009958,
+ "last": 3605442.877838291,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.763545246040287,
+ "seconds": 12.281139708,
+ "prefill_seconds": 0.099029959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.060062583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534515041,
+ "first": 3605430.596032833,
+ "last": 3605442.877956,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.812497437850745,
+ "seconds": 12.870664125,
+ "prefill_seconds": 0.096127583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.470914666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534545208,
+ "first": 3605430.006379833,
+ "last": 3605442.876971416,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.116567752228978,
+ "seconds": 12.676118667,
+ "prefill_seconds": 0.097671334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.665405875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.5342855,
+ "first": 3605430.200931208,
+ "last": 3605442.877004958,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.647095139435265,
+ "seconds": 11.779871542,
+ "prefill_seconds": 0.10088975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.561067417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534324333,
+ "first": 3605431.097468208,
+ "last": 3605442.877751208,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.46325171461922,
+ "seconds": 11.880772,
+ "prefill_seconds": 0.100381917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.460275792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.5345785,
+ "first": 3605430.99657175,
+ "last": 3605442.8774815,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.43321193835248,
+ "seconds": 12.479682625,
+ "prefill_seconds": 0.098590583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.861649792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.534357583,
+ "first": 3605430.397450916,
+ "last": 3605442.8771395,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.273075142073512,
+ "seconds": 12.5782595,
+ "prefill_seconds": 0.097901292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.763198375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605429.533873541,
+ "first": 3605430.29883175,
+ "last": 3605442.877249875,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 145.3755917478066,
+ "ttft_max": 0.09841450024396181,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 145.36730782270743,
+ "seconds": 1.754177083,
+ "prefill_seconds": 0.097816084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098008292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 78,
+ "drafted": 285,
+ "accepted": 177,
+ "acceptance_rate": 0.6210526315789474,
+ "tokens_per_round": 3.282051282051282
+ },
+ "sent": 3605442.883334708,
+ "first": 3605442.981749208,
+ "last": 3605444.735826333,
+ "chunks": 79
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 218.4538282582019,
+ "ttft_max": 0.44721245765686035,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 57.43069322667104,
+ "seconds": 4.440134459,
+ "prefill_seconds": 0.11613075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.330156041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 11,
+ "accepted": 4,
+ "acceptance_rate": 0.36363636363636365,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605444.741474708,
+ "first": 3605445.072298333,
+ "last": 3605449.512541,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.61526033658863,
+ "seconds": 4.669024709,
+ "prefill_seconds": 0.101002,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.101313875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 11,
+ "accepted": 4,
+ "acceptance_rate": 0.36363636363636365,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605444.741541416,
+ "first": 3605444.843362125,
+ "last": 3605449.51230525,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 55.96697684991846,
+ "seconds": 4.55625825,
+ "prefill_seconds": 0.112759209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.214062,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 11,
+ "accepted": 4,
+ "acceptance_rate": 0.36363636363636365,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605444.741598166,
+ "first": 3605444.9561535,
+ "last": 3605449.512485458,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 58.97507051495962,
+ "seconds": 4.323860875,
+ "prefill_seconds": 0.116266625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.446445666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 11,
+ "accepted": 4,
+ "acceptance_rate": 0.36363636363636365,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 3605444.741394,
+ "first": 3605445.188606458,
+ "last": 3605449.512434791,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 276.2903641691943,
+ "ttft_max": 0.8495748750865459,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.54092742950388,
+ "seconds": 7.382546416,
+ "prefill_seconds": 0.099146834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.099410292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 63,
+ "accepted": 60,
+ "acceptance_rate": 0.9523809523809523,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605449.518375708,
+ "first": 3605449.618810166,
+ "last": 3605457.001380083,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 37.823851195836035,
+ "seconds": 6.741777792,
+ "prefill_seconds": 0.108070917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.740279375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 63,
+ "accepted": 60,
+ "acceptance_rate": 0.9523809523809523,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605449.518509833,
+ "first": 3605450.259891375,
+ "last": 3605457.002291791,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.09481406821825,
+ "seconds": 7.064726792,
+ "prefill_seconds": 0.106603333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.417260542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 63,
+ "accepted": 60,
+ "acceptance_rate": 0.9523809523809523,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605449.518622416,
+ "first": 3605449.936744791,
+ "last": 3605457.002059708,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.03749687564972,
+ "seconds": 7.277917167,
+ "prefill_seconds": 0.104644917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2040505,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 63,
+ "accepted": 60,
+ "acceptance_rate": 0.9523809523809523,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605449.518439166,
+ "first": 3605449.72347175,
+ "last": 3605457.002032125,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.649998499347845,
+ "seconds": 6.957708334,
+ "prefill_seconds": 0.107106167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.524294333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 63,
+ "accepted": 60,
+ "acceptance_rate": 0.9523809523809523,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605449.51878775,
+ "first": 3605450.043845541,
+ "last": 3605457.002192833,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 38.441681761612095,
+ "seconds": 6.633424666,
+ "prefill_seconds": 0.108358833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.848550417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 63,
+ "accepted": 60,
+ "acceptance_rate": 0.9523809523809523,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605449.518678541,
+ "first": 3605450.368253416,
+ "last": 3605457.002250708,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 37.227168873286004,
+ "seconds": 6.849835959,
+ "prefill_seconds": 0.107952625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.632268834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 63,
+ "accepted": 60,
+ "acceptance_rate": 0.9523809523809523,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605449.518562333,
+ "first": 3605450.151802666,
+ "last": 3605457.002346708,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.55849484955847,
+ "seconds": 7.171282167,
+ "prefill_seconds": 0.106634083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.310710417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 63,
+ "accepted": 60,
+ "acceptance_rate": 0.9523809523809523,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605449.51828825,
+ "first": 3605449.830112083,
+ "last": 3605457.001886833,
+ "chunks": 196
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 305.7837592707918,
+ "ttft_max": 1.7033216250129044,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.227481228525594,
+ "seconds": 12.606611625,
+ "prefill_seconds": 0.107409542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.833712583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.00861875,
+ "first": 3605457.844106125,
+ "last": 3605470.450826708,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.328889265817107,
+ "seconds": 11.955615542,
+ "prefill_seconds": 0.109789625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.484054917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.008686833,
+ "first": 3605458.4952175,
+ "last": 3605470.450810958,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.567234730667614,
+ "seconds": 13.031989625,
+ "prefill_seconds": 0.104422084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.408458958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.00904975,
+ "first": 3605457.418656166,
+ "last": 3605470.450537833,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.134836567906717,
+ "seconds": 12.065387834,
+ "prefill_seconds": 0.108807208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.374314042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.009264791,
+ "first": 3605458.385420666,
+ "last": 3605470.450794625,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.580029918216074,
+ "seconds": 12.390652541,
+ "prefill_seconds": 0.108638917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.0496385,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.008748625,
+ "first": 3605458.060122333,
+ "last": 3605470.45078075,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.726250061702938,
+ "seconds": 12.926937416,
+ "prefill_seconds": 0.105059,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.513551125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.008799458,
+ "first": 3605457.5237645,
+ "last": 3605470.450574833,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.111733770131554,
+ "seconds": 13.342588541,
+ "prefill_seconds": 0.098031875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098241875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.009110625,
+ "first": 3605457.108064458,
+ "last": 3605470.4505095,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.411757826194172,
+ "seconds": 13.136368292,
+ "prefill_seconds": 0.103489208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.304175709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.008959458,
+ "first": 3605457.314250791,
+ "last": 3605470.450658833,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.761268916543667,
+ "seconds": 12.28248625,
+ "prefill_seconds": 0.108184833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.15796075,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.009216583,
+ "first": 3605458.168288208,
+ "last": 3605470.450773291,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.723004468900438,
+ "seconds": 11.738707708,
+ "prefill_seconds": 0.108524208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.700951291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.008847083,
+ "first": 3605458.712168708,
+ "last": 3605470.450819875,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.945919433429196,
+ "seconds": 12.174208958,
+ "prefill_seconds": 0.10830875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.265652833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.009310833,
+ "first": 3605458.276619291,
+ "last": 3605470.450755083,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.889079449391613,
+ "seconds": 12.821106208,
+ "prefill_seconds": 0.105834375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.619311958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.009351708,
+ "first": 3605457.629581875,
+ "last": 3605470.450625041,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.40119374239388,
+ "seconds": 12.499268583,
+ "prefill_seconds": 0.107347,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.940954208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.0090025,
+ "first": 3605457.951464666,
+ "last": 3605470.450725125,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.260036328797145,
+ "seconds": 13.239850416,
+ "prefill_seconds": 0.102721667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.200804834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.008900416,
+ "first": 3605457.210753041,
+ "last": 3605470.450522791,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.056534521066187,
+ "seconds": 12.714060833,
+ "prefill_seconds": 0.107087416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.726305417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.009167583,
+ "first": 3605457.736689583,
+ "last": 3605470.450684333,
+ "chunks": 254
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.524041935164966,
+ "seconds": 11.847217208,
+ "prefill_seconds": 0.108408375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.592457708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 253,
+ "drafted": 2,
+ "accepted": 2,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.0118577075098814
+ },
+ "sent": 3605457.008531416,
+ "first": 3605458.603612666,
+ "last": 3605470.450765791,
+ "chunks": 254
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 134.04015127905572,
+ "ttft_max": 0.08502129185944796,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 134.03280939207315,
+ "seconds": 1.90251925,
+ "prefill_seconds": 0.084423209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.084615291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 204,
+ "drafted": 90,
+ "accepted": 51,
+ "acceptance_rate": 0.5666666666666667,
+ "tokens_per_round": 1.2549019607843137
+ },
+ "sent": 3605475.610554166,
+ "first": 3605475.695575458,
+ "last": 3605477.5979905,
+ "chunks": 205
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 245.54320777721466,
+ "ttft_max": 0.34452699963003397,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 64.03927407833346,
+ "seconds": 3.981931458,
+ "prefill_seconds": 0.085632083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.25774,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605477.614962666,
+ "first": 3605477.873537583,
+ "last": 3605481.855391875,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 65.45556544127109,
+ "seconds": 3.895772625,
+ "prefill_seconds": 0.086155875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.343880375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605477.6151625,
+ "first": 3605477.9596895,
+ "last": 3605481.855459041,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 62.691247363564464,
+ "seconds": 4.067553458,
+ "prefill_seconds": 0.086513208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.172089333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605477.615059958,
+ "first": 3605477.787903041,
+ "last": 3605481.855448625,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 61.385409103510334,
+ "seconds": 4.154081625,
+ "prefill_seconds": 0.085394667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.085709541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605477.614839208,
+ "first": 3605477.7014425,
+ "last": 3605481.855497541,
+ "chunks": 216
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 270.7423117196063,
+ "ttft_max": 0.7330167503096163,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.84593855412964,
+ "seconds": 7.534138833,
+ "prefill_seconds": 0.0900295,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.090248,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605481.861976458,
+ "first": 3605481.953038875,
+ "last": 3605489.487249041,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.1149320635995,
+ "seconds": 7.261867958,
+ "prefill_seconds": 0.090984167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.362650291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605481.862190875,
+ "first": 3605482.225547375,
+ "last": 3605489.487622208,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.562696398712745,
+ "seconds": 7.170434917,
+ "prefill_seconds": 0.091634833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.454281416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605481.862037,
+ "first": 3605482.317202208,
+ "last": 3605489.487771833,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.99559208154767,
+ "seconds": 6.892713041,
+ "prefill_seconds": 0.092998417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.732097667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605481.862138625,
+ "first": 3605482.595155375,
+ "last": 3605489.487726291,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.68031094366792,
+ "seconds": 7.35287525,
+ "prefill_seconds": 0.090826291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.271782708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605481.862229875,
+ "first": 3605482.134551708,
+ "last": 3605489.487819208,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.02547021935962,
+ "seconds": 7.078325375,
+ "prefill_seconds": 0.092130791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.546408625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605481.862090458,
+ "first": 3605482.409339833,
+ "last": 3605489.487566125,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.503284092463026,
+ "seconds": 6.985672833,
+ "prefill_seconds": 0.092687833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.6390715,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605481.862369958,
+ "first": 3605482.502132416,
+ "last": 3605489.48766875,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.258055273838764,
+ "seconds": 7.443504833,
+ "prefill_seconds": 0.090699459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.180952834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605481.861897625,
+ "first": 3605482.04376525,
+ "last": 3605489.487878875,
+ "chunks": 229
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 292.93412642405116,
+ "ttft_max": 1.4647241248749197,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.926920256904737,
+ "seconds": 13.472873375,
+ "prefill_seconds": 0.09103375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.546364292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494451375,
+ "first": 3605490.042471791,
+ "last": 3605503.515362625,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.733000571158392,
+ "seconds": 12.922515209,
+ "prefill_seconds": 0.091383459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.096374833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494896916,
+ "first": 3605490.59284475,
+ "last": 3605503.5152445,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.550024820973466,
+ "seconds": 13.746612334,
+ "prefill_seconds": 0.090700209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.272539583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494518625,
+ "first": 3605489.76867875,
+ "last": 3605503.515130208,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.456901603825724,
+ "seconds": 13.105889375,
+ "prefill_seconds": 0.091986333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.91311975,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494632416,
+ "first": 3605490.409464791,
+ "last": 3605503.515424291,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.308820640498432,
+ "seconds": 13.927713041,
+ "prefill_seconds": 0.091075125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0913155,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.49495075,
+ "first": 3605489.587432416,
+ "last": 3605503.515098,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.594456981695206,
+ "seconds": 13.0138845,
+ "prefill_seconds": 0.091970959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.005073208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494578333,
+ "first": 3605490.501446208,
+ "last": 3605503.515478458,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.30949796549612,
+ "seconds": 12.555701792,
+ "prefill_seconds": 0.091528292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.462739375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494995833,
+ "first": 3605490.959719958,
+ "last": 3605503.515442125,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.056723214752623,
+ "seconds": 13.38110425,
+ "prefill_seconds": 0.091652,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.637867125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.495036,
+ "first": 3605490.134127541,
+ "last": 3605503.5153105,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.67458408863247,
+ "seconds": 13.654922583,
+ "prefill_seconds": 0.091694166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.364163875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494694291,
+ "first": 3605489.860316083,
+ "last": 3605503.515465708,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.321338124166765,
+ "seconds": 13.197843667,
+ "prefill_seconds": 0.091791208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.821177041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494736916,
+ "first": 3605490.3174595,
+ "last": 3605503.515433416,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.80005716768568,
+ "seconds": 13.563788542,
+ "prefill_seconds": 0.091140583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.455208083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.49508525,
+ "first": 3605489.951429166,
+ "last": 3605503.515194958,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.187885356109707,
+ "seconds": 13.289635375,
+ "prefill_seconds": 0.091527333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.729372166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.49484925,
+ "first": 3605490.225666791,
+ "last": 3605503.515388125,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.162431911887484,
+ "seconds": 12.647283875,
+ "prefill_seconds": 0.091754458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.3712595,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494791166,
+ "first": 3605490.868178291,
+ "last": 3605503.515450375,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.01728296135841,
+ "seconds": 12.738991625,
+ "prefill_seconds": 0.0919845,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.279713875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.495518708,
+ "first": 3605490.776427416,
+ "last": 3605503.515411041,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 18.428504408437014,
+ "seconds": 13.837259625,
+ "prefill_seconds": 0.090444042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.181785917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.49517225,
+ "first": 3605489.677895625,
+ "last": 3605503.515071041,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.87381691536153,
+ "seconds": 12.830952458,
+ "prefill_seconds": 0.091586458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.187832792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 29,
+ "accepted": 27,
+ "acceptance_rate": 0.9310344827586207,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605489.494358,
+ "first": 3605490.68442925,
+ "last": 3605503.515375041,
+ "chunks": 229
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 134.8330796716681,
+ "ttft_max": 0.13193579204380512,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 134.82593874550517,
+ "seconds": 1.891327458,
+ "prefill_seconds": 0.131301792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131517041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 144,
+ "drafted": 180,
+ "accepted": 111,
+ "acceptance_rate": 0.6166666666666667,
+ "tokens_per_round": 1.7777777777777777
+ },
+ "sent": 3605503.525222083,
+ "first": 3605503.657157875,
+ "last": 3605505.548385166,
+ "chunks": 145
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 225.21937567407258,
+ "ttft_max": 0.5493802912533283,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 58.06139216612445,
+ "seconds": 4.391902958,
+ "prefill_seconds": 0.136901375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.27295775,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605505.554539708,
+ "first": 3605505.828181583,
+ "last": 3605510.220012125,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 56.306342560787606,
+ "seconds": 4.528797084,
+ "prefill_seconds": 0.135906542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.13614225,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605505.554665291,
+ "first": 3605505.69124725,
+ "last": 3605510.220045833,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 61.95063353421815,
+ "seconds": 4.116180666,
+ "prefill_seconds": 0.137669792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.548800833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605505.55460375,
+ "first": 3605506.103984041,
+ "last": 3605510.220164875,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 59.94556237694263,
+ "seconds": 4.2538595,
+ "prefill_seconds": 0.138106334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.411087208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605505.5544525,
+ "first": 3605505.96628925,
+ "last": 3605510.22013575,
+ "chunks": 167
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 259.820332969546,
+ "ttft_max": 0.902233749628067,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.54192665919949,
+ "seconds": 7.174625125,
+ "prefill_seconds": 0.114212292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.785898625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605510.226858166,
+ "first": 3605511.013980875,
+ "last": 3605518.18909125,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.447547902386255,
+ "seconds": 7.402558833,
+ "prefill_seconds": 0.1132025,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.558141875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605510.226640083,
+ "first": 3605510.785963583,
+ "last": 3605518.188963833,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.985114146738646,
+ "seconds": 7.288814292,
+ "prefill_seconds": 0.1137675,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.671693375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605510.226808916,
+ "first": 3605510.899731583,
+ "last": 3605518.1890325,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.9289249370816,
+ "seconds": 7.515711166,
+ "prefill_seconds": 0.112677292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.445253042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605510.22669775,
+ "first": 3605510.672740375,
+ "last": 3605518.188532375,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.94573820483554,
+ "seconds": 7.739999584,
+ "prefill_seconds": 0.110556541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.220648625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605510.226751083,
+ "first": 3605510.448116708,
+ "last": 3605518.188665958,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.12177692157362,
+ "seconds": 7.059453375,
+ "prefill_seconds": 0.115266209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.901208792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605510.227011333,
+ "first": 3605511.129245083,
+ "last": 3605518.188815666,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.481980621250365,
+ "seconds": 7.8505065,
+ "prefill_seconds": 0.109736375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.110060875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605510.226935625,
+ "first": 3605510.33751175,
+ "last": 3605518.188263375,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.42890668679567,
+ "seconds": 7.628128625,
+ "prefill_seconds": 0.111950875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.332594958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605510.226550291,
+ "first": 3605510.560035833,
+ "last": 3605518.188732541,
+ "chunks": 196
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 306.84608203546827,
+ "ttft_max": 1.7599479579366744,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.17835935085708,
+ "seconds": 13.296236416,
+ "prefill_seconds": 0.107219209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.107445625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.195807791,
+ "first": 3605518.304832041,
+ "last": 3605531.601008625,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.832565639618313,
+ "seconds": 12.857640541,
+ "prefill_seconds": 0.110754666,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.54585225,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.195857416,
+ "first": 3605518.743459541,
+ "last": 3605531.601181958,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.897513191293374,
+ "seconds": 11.645157958,
+ "prefill_seconds": 0.108737208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.757228209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196149458,
+ "first": 3605519.956097416,
+ "last": 3605531.601200083,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.36116420187307,
+ "seconds": 12.523841833,
+ "prefill_seconds": 0.111492834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.879644042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.195917958,
+ "first": 3605519.077346291,
+ "last": 3605531.601344041,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.497621232863647,
+ "seconds": 13.0785185,
+ "prefill_seconds": 0.109325875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.325160334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196680708,
+ "first": 3605518.522573291,
+ "last": 3605531.601083916,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.33601166000266,
+ "seconds": 13.187828208,
+ "prefill_seconds": 0.108400667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.215765959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.195975708,
+ "first": 3605518.413255375,
+ "last": 3605531.601117,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.694897027351384,
+ "seconds": 11.753916125,
+ "prefill_seconds": 0.108971333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.648525292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196330583,
+ "first": 3605519.847375333,
+ "last": 3605531.601384875,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.18146261034833,
+ "seconds": 12.63535775,
+ "prefill_seconds": 0.111334958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.76831725,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.1960735,
+ "first": 3605518.965816791,
+ "last": 3605531.601354958,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.299143038827552,
+ "seconds": 11.972312667,
+ "prefill_seconds": 0.109716708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.430249917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196623625,
+ "first": 3605519.628945291,
+ "last": 3605531.601218833,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.105705712451527,
+ "seconds": 12.082040917,
+ "prefill_seconds": 0.110033333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.320596875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196563083,
+ "first": 3605519.519230083,
+ "last": 3605531.601266041,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.91525583241108,
+ "seconds": 12.192057417,
+ "prefill_seconds": 0.1104875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.210580166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196260583,
+ "first": 3605519.409184083,
+ "last": 3605531.601251291,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.663188221835757,
+ "seconds": 12.968395416,
+ "prefill_seconds": 0.110114292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.435185958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196021708,
+ "first": 3605518.632716416,
+ "last": 3605531.60104925,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.72746324258728,
+ "seconds": 12.3025185,
+ "prefill_seconds": 0.110307333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.100176625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.19639025,
+ "first": 3605519.298645125,
+ "last": 3605531.601400708,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.00525320846724,
+ "seconds": 12.746651959,
+ "prefill_seconds": 0.110993667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.656783625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196212041,
+ "first": 3605518.854473916,
+ "last": 3605531.601234666,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.495638724570185,
+ "seconds": 11.8628715,
+ "prefill_seconds": 0.109426,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.539636958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.196869666,
+ "first": 3605519.738365708,
+ "last": 3605531.601364166,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.543276273335106,
+ "seconds": 12.412820458,
+ "prefill_seconds": 0.111021292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.990023,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 18,
+ "accepted": 18,
+ "acceptance_rate": 1.0,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 3605518.195726041,
+ "first": 3605519.188370916,
+ "last": 3605531.6011585,
+ "chunks": 238
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 131.02762955084603,
+ "ttft_max": 0.09522395813837647,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 131.01969958356412,
+ "seconds": 1.946272208,
+ "prefill_seconds": 0.094651084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094815916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 204,
+ "drafted": 90,
+ "accepted": 51,
+ "acceptance_rate": 0.5666666666666667,
+ "tokens_per_round": 1.2549019607843137
+ },
+ "sent": 3605536.725807625,
+ "first": 3605536.821031583,
+ "last": 3605538.767186,
+ "chunks": 205
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 223.55292221131992,
+ "ttft_max": 0.41851866664364934,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 57.185937215559534,
+ "seconds": 4.45913825,
+ "prefill_seconds": 0.103478292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.205440625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605538.77615325,
+ "first": 3605538.982185041,
+ "last": 3605543.441316208,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 60.04708515180622,
+ "seconds": 4.246667417,
+ "prefill_seconds": 0.107146875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.417955625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605538.776206083,
+ "first": 3605539.19472475,
+ "last": 3605543.441306208,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 55.88888894368192,
+ "seconds": 4.56262425,
+ "prefill_seconds": 0.101644542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.101953875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605538.77628525,
+ "first": 3605538.87870325,
+ "last": 3605543.441272791,
+ "chunks": 216
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 58.56940889280687,
+ "seconds": 4.353808666,
+ "prefill_seconds": 0.105337417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.310781958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 215,
+ "drafted": 51,
+ "accepted": 40,
+ "acceptance_rate": 0.7843137254901961,
+ "tokens_per_round": 1.1906976744186046
+ },
+ "sent": 3605538.77606675,
+ "first": 3605539.087549666,
+ "last": 3605543.44138125,
+ "chunks": 216
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 267.110955393874,
+ "ttft_max": 0.7607742501422763,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.088886715693185,
+ "seconds": 7.065887125,
+ "prefill_seconds": 0.096627542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.663958834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605543.448670208,
+ "first": 3605544.11365275,
+ "last": 3605551.179667666,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.120935183051415,
+ "seconds": 7.260626708,
+ "prefill_seconds": 0.095467708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.469219375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605543.448989833,
+ "first": 3605543.918952083,
+ "last": 3605551.179549,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.80224652784556,
+ "seconds": 7.543877292,
+ "prefill_seconds": 0.093102875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.185995834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605543.448920625,
+ "first": 3605543.635653291,
+ "last": 3605551.1797455,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.39017811252395,
+ "seconds": 7.636976333,
+ "prefill_seconds": 0.092668541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.092922833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605543.448741083,
+ "first": 3605543.542469833,
+ "last": 3605551.179358875,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.60209569225996,
+ "seconds": 7.162499708,
+ "prefill_seconds": 0.098108875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.567303125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605543.448808416,
+ "first": 3605544.017031583,
+ "last": 3605551.179516375,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.66521645381622,
+ "seconds": 7.356076958,
+ "prefill_seconds": 0.094272583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.373741791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605543.448862708,
+ "first": 3605543.823425708,
+ "last": 3605551.1794025,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.586875211863024,
+ "seconds": 6.969712459,
+ "prefill_seconds": 0.096179833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.760045167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605543.44908225,
+ "first": 3605544.2098565,
+ "last": 3605551.179606166,
+ "chunks": 229
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.22654626729075,
+ "seconds": 7.450357334,
+ "prefill_seconds": 0.093555,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.279560833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 228,
+ "drafted": 30,
+ "accepted": 27,
+ "acceptance_rate": 0.9,
+ "tokens_per_round": 1.1228070175438596
+ },
+ "sent": 3605543.448587083,
+ "first": 3605543.729159208,
+ "last": 3605551.179385541,
+ "chunks": 229
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 304.5591714806901,
+ "ttft_max": 1.5759439589455724,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.896696675450308,
+ "seconds": 12.816197792,
+ "prefill_seconds": 0.098274708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.673495459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187829666,
+ "first": 3605551.862587583,
+ "last": 3605564.679362333,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.454755059424507,
+ "seconds": 13.107335416,
+ "prefill_seconds": 0.096827833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.381084167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187347875,
+ "first": 3605551.5699955,
+ "last": 3605564.678450041,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.173570848218482,
+ "seconds": 13.299557084,
+ "prefill_seconds": 0.094394833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.188357292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187417791,
+ "first": 3605551.377271083,
+ "last": 3605564.676923666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.401400417421726,
+ "seconds": 11.915108125,
+ "prefill_seconds": 0.100877667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.57352325,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187671166,
+ "first": 3605552.763615125,
+ "last": 3605564.679279708,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.59868609461536,
+ "seconds": 13.011076292,
+ "prefill_seconds": 0.097481084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.478319084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187617791,
+ "first": 3605551.667259833,
+ "last": 3605564.678642791,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.701212860267265,
+ "seconds": 12.318118833,
+ "prefill_seconds": 0.100245917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.170799667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.18805875,
+ "first": 3605552.3602425,
+ "last": 3605564.679030583,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.745475464432126,
+ "seconds": 12.914350959,
+ "prefill_seconds": 0.097031708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.575222417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187489,
+ "first": 3605551.764302416,
+ "last": 3605564.678928875,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.312291263568483,
+ "seconds": 13.204026209,
+ "prefill_seconds": 0.095629917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.284033292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187895708,
+ "first": 3605551.472959958,
+ "last": 3605564.676970541,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.044453199714567,
+ "seconds": 12.117207208,
+ "prefill_seconds": 0.100306625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.371618125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187939375,
+ "first": 3605552.5613415,
+ "last": 3605564.678822625,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.53412080141582,
+ "seconds": 12.418354916,
+ "prefill_seconds": 0.099302875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.070665625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187554916,
+ "first": 3605552.259993458,
+ "last": 3605564.678601375,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.053464962880685,
+ "seconds": 12.716006958,
+ "prefill_seconds": 0.0985685,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.772066042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187994291,
+ "first": 3605551.96115275,
+ "last": 3605564.677346375,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.87184934034128,
+ "seconds": 12.217412834,
+ "prefill_seconds": 0.100775084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.271502834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.188120791,
+ "first": 3605552.4610205,
+ "last": 3605564.679188333,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.211309971776885,
+ "seconds": 12.616698292,
+ "prefill_seconds": 0.099337709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.871298708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.188180583,
+ "first": 3605552.06051425,
+ "last": 3605564.677739375,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.03846760643467,
+ "seconds": 13.393935125,
+ "prefill_seconds": 0.093820291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094052667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187727583,
+ "first": 3605551.282950791,
+ "last": 3605564.678359958,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.22169114824537,
+ "seconds": 12.0160075,
+ "prefill_seconds": 0.101354042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.472934666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187773083,
+ "first": 3605552.6626945,
+ "last": 3605564.67910925,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.37291427891917,
+ "seconds": 12.516618708,
+ "prefill_seconds": 0.100147458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.971464084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f66e3fd7e509",
+ "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605551.187247375,
+ "first": 3605552.160674291,
+ "last": 3605564.67800675,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 154.95886207861588,
+ "ttft_max": 0.10082387505099177,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 154.94833976171662,
+ "seconds": 1.645709792,
+ "prefill_seconds": 0.099591459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.100275959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 144,
+ "drafted": 180,
+ "accepted": 111,
+ "acceptance_rate": 0.6166666666666667,
+ "tokens_per_round": 1.7777777777777777
+ },
+ "sent": 3605564.688457666,
+ "first": 3605564.789281541,
+ "last": 3605566.434879583,
+ "chunks": 145
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 242.36047700354084,
+ "ttft_max": 0.42187445890158415,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 60.59083789787342,
+ "seconds": 4.208557083,
+ "prefill_seconds": 0.102118834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.102311542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605566.441818333,
+ "first": 3605566.544721125,
+ "last": 3605570.753210666,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 65.55965406317821,
+ "seconds": 3.889587333,
+ "prefill_seconds": 0.1074365,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.421266333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605566.441888166,
+ "first": 3605566.863762625,
+ "last": 3605570.753328125,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 62.141115395800036,
+ "seconds": 4.103563291,
+ "prefill_seconds": 0.105009958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2073145,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605566.441946958,
+ "first": 3605566.649737666,
+ "last": 3605570.753248791,
+ "chunks": 167
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 63.79769276269591,
+ "seconds": 3.99700975,
+ "prefill_seconds": 0.106556875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.313817375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 166,
+ "drafted": 99,
+ "accepted": 89,
+ "acceptance_rate": 0.898989898989899,
+ "tokens_per_round": 1.5421686746987953
+ },
+ "sent": 3605566.441734333,
+ "first": 3605566.756301208,
+ "last": 3605570.753289875,
+ "chunks": 167
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 263.8200654034127,
+ "ttft_max": 0.9031712496653199,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.962132787976955,
+ "seconds": 7.508362375,
+ "prefill_seconds": 0.112015833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.335270667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605570.760726458,
+ "first": 3605571.09699175,
+ "last": 3605578.605317541,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.573171744850306,
+ "seconds": 7.168323416,
+ "prefill_seconds": 0.114088625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.675248375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605570.760784666,
+ "first": 3605571.437124833,
+ "last": 3605578.605774666,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 35.01562509172935,
+ "seconds": 7.282463167,
+ "prefill_seconds": 0.112924792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.561342583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605570.760840583,
+ "first": 3605571.323135333,
+ "last": 3605578.605375083,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.14625113408863,
+ "seconds": 7.0546735,
+ "prefill_seconds": 0.113693125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.788898458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605570.761060916,
+ "first": 3605571.550822083,
+ "last": 3605578.605628708,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 34.481382512777,
+ "seconds": 7.395295125,
+ "prefill_seconds": 0.11306225,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.448364625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605570.7608885,
+ "first": 3605571.210063208,
+ "last": 3605578.605350541,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 36.73751667190106,
+ "seconds": 6.941133291,
+ "prefill_seconds": 0.113508459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.902484542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605570.761161208,
+ "first": 3605571.664332458,
+ "last": 3605578.605549208,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 32.97946084302359,
+ "seconds": 7.732085167,
+ "prefill_seconds": 0.111460375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.111665417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605570.760939416,
+ "first": 3605570.873231666,
+ "last": 3605578.605298208,
+ "chunks": 196
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 33.462928365466354,
+ "seconds": 7.620373125,
+ "prefill_seconds": 0.111713334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.22331875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 195,
+ "drafted": 62,
+ "accepted": 60,
+ "acceptance_rate": 0.967741935483871,
+ "tokens_per_round": 1.3128205128205128
+ },
+ "sent": 3605570.760643791,
+ "first": 3605570.98495875,
+ "last": 3605578.605269208,
+ "chunks": 196
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 310.3552343905495,
+ "ttft_max": 1.649341749958694,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.601231295597064,
+ "seconds": 11.804882625,
+ "prefill_seconds": 0.103076792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.438742334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.613517666,
+ "first": 3605580.054471916,
+ "last": 3605591.860033416,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.514480305695376,
+ "seconds": 12.430244208,
+ "prefill_seconds": 0.106008791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.813621208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.6139825,
+ "first": 3605579.428936916,
+ "last": 3605591.859972666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.17141942444813,
+ "seconds": 12.641648792,
+ "prefill_seconds": 0.103951,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.602347542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.613593041,
+ "first": 3605579.217488416,
+ "last": 3605591.859112083,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.69574863463283,
+ "seconds": 12.946956459,
+ "prefill_seconds": 0.0992075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.297069625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.613781833,
+ "first": 3605578.912178,
+ "last": 3605591.860200125,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.41421071156215,
+ "seconds": 11.907980333,
+ "prefill_seconds": 0.104237667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.3356565,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.6137215,
+ "first": 3605579.951374291,
+ "last": 3605591.86023075,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.045202774397332,
+ "seconds": 12.116775625,
+ "prefill_seconds": 0.104969083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.12696575,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.614247583,
+ "first": 3605579.742488625,
+ "last": 3605591.86000325,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.688762015476815,
+ "seconds": 12.325532084,
+ "prefill_seconds": 0.104711167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.918210667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.613651375,
+ "first": 3605579.533632958,
+ "last": 3605591.859162416,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.795983017424774,
+ "seconds": 11.6994035,
+ "prefill_seconds": 0.105472208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.544115209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.614026916,
+ "first": 3605580.159944208,
+ "last": 3605591.859847833,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.341035948504953,
+ "seconds": 12.536234666,
+ "prefill_seconds": 0.10541625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.707677917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.613929291,
+ "first": 3605579.322897208,
+ "last": 3605591.859859708,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.398889956353763,
+ "seconds": 13.145082042,
+ "prefill_seconds": 0.098952833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.099173125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.613878041,
+ "first": 3605578.714004916,
+ "last": 3605591.859769666,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.228452308918147,
+ "seconds": 12.012180459,
+ "prefill_seconds": 0.104630334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.231449042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.614070458,
+ "first": 3605579.847168583,
+ "last": 3605591.859674458,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.849658943156147,
+ "seconds": 12.846568333,
+ "prefill_seconds": 0.100359583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.397427875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.613832041,
+ "first": 3605579.012549083,
+ "last": 3605591.860064916,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 21.990573014412327,
+ "seconds": 11.595877917,
+ "prefill_seconds": 0.103537833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.647648167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.614167625,
+ "first": 3605580.263509375,
+ "last": 3605591.859724208,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.864530693861955,
+ "seconds": 12.221698333,
+ "prefill_seconds": 0.103841959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.022063,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.614118708,
+ "first": 3605579.637514458,
+ "last": 3605591.859193,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 20.006909837448564,
+ "seconds": 12.7455965,
+ "prefill_seconds": 0.100966709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.498359167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.614321291,
+ "first": 3605579.113519083,
+ "last": 3605591.859611208,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 19.54596923465485,
+ "seconds": 13.046168084,
+ "prefill_seconds": 0.098901375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.197920209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "5a6a547bd3c4",
+ "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 3605578.613433208,
+ "first": 3605578.812915958,
+ "last": 3605591.859001875,
+ "chunks": 256
+ }
+ ]
+ }
+ ],
+ "performance-qwen": [
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 30.344307936698552,
+ "ttft_max": 0.23980883276090026,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 30.344048558503754,
+ "seconds": 8.403624833,
+ "prefill_seconds": 0.239020458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.239324791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 55,
+ "drafted": 335,
+ "accepted": 200,
+ "acceptance_rate": 0.5970149253731343,
+ "tokens_per_round": 4.654545454545454
+ },
+ "sent": 3604496.127048333,
+ "first": 3604496.366857166,
+ "last": 3604504.770410166,
+ "chunks": 56
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 30.93870088644765,
+ "ttft_max": 25.042438166681677,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 31.830604264701808,
+ "seconds": 8.011158,
+ "prefill_seconds": 0.229463833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 8.644388292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 329,
+ "accepted": 202,
+ "acceptance_rate": 0.6139817629179332,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3604504.77142875,
+ "first": 3604513.416526083,
+ "last": 3604521.427605083,
+ "chunks": 54
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 31.158479446950714,
+ "seconds": 8.183968041,
+ "prefill_seconds": 0.230723791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.231085125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 55,
+ "drafted": 327,
+ "accepted": 200,
+ "acceptance_rate": 0.6116207951070336,
+ "tokens_per_round": 4.654545454545454
+ },
+ "sent": 3604504.771560125,
+ "first": 3604505.003225333,
+ "last": 3604513.187041583,
+ "chunks": 56
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 32.17277731613354,
+ "seconds": 7.925955459,
+ "prefill_seconds": 0.23254725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 16.888045542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 323,
+ "accepted": 202,
+ "acceptance_rate": 0.6253869969040248,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3604504.771489125,
+ "first": 3604521.660191041,
+ "last": 3604529.586075875,
+ "chunks": 54
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 31.25806670338704,
+ "seconds": 8.157894166,
+ "prefill_seconds": 0.227744292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 25.041650917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 54,
+ "drafted": 328,
+ "accepted": 201,
+ "acceptance_rate": 0.6128048780487805,
+ "tokens_per_round": 4.7407407407407405
+ },
+ "sent": 3604504.771343333,
+ "first": 3604529.8137815,
+ "last": 3604537.971642583,
+ "chunks": 55
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 23.065841730942,
+ "ttft_max": 0.43003095779567957,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.06554077318274,
+ "seconds": 11.055452916,
+ "prefill_seconds": 0.429262125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.429609584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 82,
+ "drafted": 345,
+ "accepted": 173,
+ "acceptance_rate": 0.5014492753623189,
+ "tokens_per_round": 3.1219512195121952
+ },
+ "sent": 3604537.973906,
+ "first": 3604538.403936958,
+ "last": 3604549.459245625,
+ "chunks": 83
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 22.55082188387638,
+ "ttft_max": 34.96590099995956,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 22.962253892898396,
+ "seconds": 11.105181625,
+ "prefill_seconds": 0.425930625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 11.91523375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 83,
+ "drafted": 356,
+ "accepted": 172,
+ "acceptance_rate": 0.48314606741573035,
+ "tokens_per_round": 3.0843373493975905
+ },
+ "sent": 3604549.460459541,
+ "first": 3604561.376253166,
+ "last": 3604572.48135725,
+ "chunks": 84
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.04879780747181,
+ "seconds": 11.06348375,
+ "prefill_seconds": 0.425502417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.425864083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 83,
+ "drafted": 356,
+ "accepted": 172,
+ "acceptance_rate": 0.48314606741573035,
+ "tokens_per_round": 3.0843373493975905
+ },
+ "sent": 3604549.460593666,
+ "first": 3604549.88709,
+ "last": 3604560.950315375,
+ "chunks": 84
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.037278457924852,
+ "seconds": 11.069015833,
+ "prefill_seconds": 0.430014417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 23.45031175,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 81,
+ "drafted": 360,
+ "accepted": 174,
+ "acceptance_rate": 0.48333333333333334,
+ "tokens_per_round": 3.1604938271604937
+ },
+ "sent": 3604549.4605245,
+ "first": 3604572.911377833,
+ "last": 3604583.980308791,
+ "chunks": 82
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.84952037711046,
+ "seconds": 10.692038916,
+ "prefill_seconds": 0.445968,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 34.965197042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 81,
+ "drafted": 348,
+ "accepted": 174,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 3.1604938271604937
+ },
+ "sent": 3604549.460381583,
+ "first": 3604584.426282583,
+ "last": 3604595.118257416,
+ "chunks": 82
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 31.3400196637318,
+ "ttft_max": 0.22861679224297404,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 31.33964026993998,
+ "seconds": 8.136660083,
+ "prefill_seconds": 0.227769375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.228073333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 375,
+ "accepted": 202,
+ "acceptance_rate": 0.5386666666666666,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3604649.728608666,
+ "first": 3604649.957225458,
+ "last": 3604658.093787041,
+ "chunks": 54
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 48.70154286002622,
+ "ttft_max": 0.8868492497131228,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.442917404598337,
+ "seconds": 20.493586167,
+ "prefill_seconds": 0.660301083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.66058425,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 217,
+ "accepted": 188,
+ "acceptance_rate": 0.8663594470046083,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 3604658.0960795,
+ "first": 3604658.757237458,
+ "last": 3604679.250707458,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.581526389590532,
+ "seconds": 20.267811083,
+ "prefill_seconds": 0.886135833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.886447709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 217,
+ "accepted": 188,
+ "acceptance_rate": 0.8663594470046083,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 3604658.096224625,
+ "first": 3604658.983073875,
+ "last": 3604679.250772125,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.308087453809105,
+ "seconds": 20.718084833,
+ "prefill_seconds": 0.435774209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.436020416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 218,
+ "accepted": 188,
+ "acceptance_rate": 0.8623853211009175,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 3604658.096145875,
+ "first": 3604658.532716958,
+ "last": 3604679.2507495,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.175402937052157,
+ "seconds": 20.943865375,
+ "prefill_seconds": 0.209930417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.210243625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 220,
+ "accepted": 188,
+ "acceptance_rate": 0.8545454545454545,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 3604658.095991625,
+ "first": 3604658.306877125,
+ "last": 3604679.250664625,
+ "chunks": 68
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 24.00490857949334,
+ "ttft_max": 0.4215462077409029,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 24.004729686343946,
+ "seconds": 10.622906541,
+ "prefill_seconds": 0.420831,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.421155459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 462,
+ "accepted": 180,
+ "acceptance_rate": 0.38961038961038963,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3604679.252754833,
+ "first": 3604679.674301041,
+ "last": 3604690.297128416,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 41.78288826705579,
+ "ttft_max": 1.6581098344177008,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.628013650766608,
+ "seconds": 23.993194625,
+ "prefill_seconds": 0.836824834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.837248625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 220,
+ "accepted": 158,
+ "acceptance_rate": 0.7181818181818181,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3604690.2980735,
+ "first": 3604691.1358885,
+ "last": 3604715.129133166,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 11.004252625792686,
+ "seconds": 23.1728595,
+ "prefill_seconds": 1.238961458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.657580292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 216,
+ "accepted": 158,
+ "acceptance_rate": 0.7314814814814815,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3604690.298201916,
+ "first": 3604691.95631175,
+ "last": 3604715.129071458,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.445826937596213,
+ "seconds": 24.411662334,
+ "prefill_seconds": 0.418261625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.418658166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 220,
+ "accepted": 158,
+ "acceptance_rate": 0.7181818181818181,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3604690.298135,
+ "first": 3604690.717330833,
+ "last": 3604715.128903916,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.813981575196546,
+ "seconds": 23.580583916,
+ "prefill_seconds": 0.831247542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.249867292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 216,
+ "accepted": 158,
+ "acceptance_rate": 0.7314814814814815,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3604690.297998291,
+ "first": 3604691.548578375,
+ "last": 3604715.12923825,
+ "chunks": 98
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 36.11988572345143,
+ "ttft_max": 0.19940866669639945,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 36.11952098561528,
+ "seconds": 7.059894291,
+ "prefill_seconds": 0.198697,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.198979292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 54,
+ "drafted": 355,
+ "accepted": 201,
+ "acceptance_rate": 0.5661971830985916,
+ "tokens_per_round": 4.7407407407407405
+ },
+ "sent": 3604725.818032958,
+ "first": 3604726.017441625,
+ "last": 3604733.077264625,
+ "chunks": 55
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 32.39324544693206,
+ "ttft_max": 20.72907174983993,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 29.33421034201612,
+ "seconds": 8.692921917,
+ "prefill_seconds": 0.205756625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.20602725,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 375,
+ "accepted": 202,
+ "acceptance_rate": 0.5386666666666666,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3604733.078777125,
+ "first": 3604733.28554225,
+ "last": 3604741.978387208,
+ "chunks": 54
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 24.466282928760503,
+ "seconds": 10.422506792,
+ "prefill_seconds": 0.24281125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 17.766874792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 62,
+ "drafted": 262,
+ "accepted": 193,
+ "acceptance_rate": 0.7366412213740458,
+ "tokens_per_round": 4.129032258064516
+ },
+ "sent": 3604733.0789195,
+ "first": 3604750.846356583,
+ "last": 3604761.2687955,
+ "chunks": 63
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.25422850288127,
+ "seconds": 10.965747583,
+ "prefill_seconds": 0.251704791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 20.728427125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 277,
+ "accepted": 188,
+ "acceptance_rate": 0.6787003610108303,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 3604733.078850041,
+ "first": 3604753.807921791,
+ "last": 3604764.773588166,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 30.390343707071754,
+ "seconds": 8.390823166,
+ "prefill_seconds": 0.234339208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 9.133264833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 368,
+ "accepted": 202,
+ "acceptance_rate": 0.5489130434782609,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3604733.078688416,
+ "first": 3604742.212719333,
+ "last": 3604750.603482375,
+ "chunks": 54
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 22.499851186148447,
+ "ttft_max": 0.4421482081525028,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 22.499807345399603,
+ "seconds": 11.333430375,
+ "prefill_seconds": 0.441396833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.44167975,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 460,
+ "accepted": 180,
+ "acceptance_rate": 0.391304347826087,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3604764.775536208,
+ "first": 3604765.217684416,
+ "last": 3604776.551092708,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 39.84032356389979,
+ "ttft_max": 1.768473374657333,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.13453022532727,
+ "seconds": 25.16150175,
+ "prefill_seconds": 0.885118083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.885429667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 220,
+ "accepted": 159,
+ "acceptance_rate": 0.7227272727272728,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3604776.552390458,
+ "first": 3604777.438389916,
+ "last": 3604802.599801291,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.3173151293953,
+ "seconds": 24.715732417,
+ "prefill_seconds": 0.886504542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.331123084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 218,
+ "accepted": 159,
+ "acceptance_rate": 0.7293577981651376,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3604776.552447875,
+ "first": 3604777.8841745,
+ "last": 3604802.599754083,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.960122290447101,
+ "seconds": 25.602095292,
+ "prefill_seconds": 0.444373291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.444644625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 220,
+ "accepted": 159,
+ "acceptance_rate": 0.7227272727272728,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3604776.55252475,
+ "first": 3604776.997629208,
+ "last": 3604802.599635708,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.502846912495857,
+ "seconds": 24.279131375,
+ "prefill_seconds": 1.323141792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.767754791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 217,
+ "accepted": 159,
+ "acceptance_rate": 0.7327188940092166,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3604776.552310333,
+ "first": 3604778.320783708,
+ "last": 3604802.599830916,
+ "chunks": 97
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 36.20655991593265,
+ "ttft_max": 0.19942687498405576,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 36.20599550053157,
+ "seconds": 7.043032417,
+ "prefill_seconds": 0.198634458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.198939541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 55,
+ "drafted": 310,
+ "accepted": 200,
+ "acceptance_rate": 0.6451612903225806,
+ "tokens_per_round": 4.654545454545454
+ },
+ "sent": 3604812.635286291,
+ "first": 3604812.834713166,
+ "last": 3604819.877635791,
+ "chunks": 56
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 47.047737556009366,
+ "ttft_max": 0.841904915869236,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.294832987952992,
+ "seconds": 20.74042,
+ "prefill_seconds": 0.411415292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.411670375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 78,
+ "drafted": 210,
+ "accepted": 177,
+ "acceptance_rate": 0.8428571428571429,
+ "tokens_per_round": 3.282051282051282
+ },
+ "sent": 3604819.879005583,
+ "first": 3604820.291204,
+ "last": 3604841.03153175,
+ "chunks": 79
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.448224614983888,
+ "seconds": 20.484848875,
+ "prefill_seconds": 0.640674458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.841417875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 79,
+ "drafted": 213,
+ "accepted": 176,
+ "acceptance_rate": 0.8262910798122066,
+ "tokens_per_round": 3.240506329113924
+ },
+ "sent": 3604819.879153375,
+ "first": 3604820.721058291,
+ "last": 3604841.205771791,
+ "chunks": 80
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 11.996807508383107,
+ "seconds": 21.255654875,
+ "prefill_seconds": 0.625017334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.625289167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 83,
+ "drafted": 219,
+ "accepted": 172,
+ "acceptance_rate": 0.7853881278538812,
+ "tokens_per_round": 3.0843373493975905
+ },
+ "sent": 3604819.879072541,
+ "first": 3604820.504795458,
+ "last": 3604841.760405958,
+ "chunks": 84
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.17110093138091,
+ "seconds": 20.951268208,
+ "prefill_seconds": 0.200510875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.200836458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 78,
+ "drafted": 212,
+ "accepted": 177,
+ "acceptance_rate": 0.8349056603773585,
+ "tokens_per_round": 3.282051282051282
+ },
+ "sent": 3604819.878924333,
+ "first": 3604820.080298625,
+ "last": 3604841.031495291,
+ "chunks": 79
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 21.34347493314762,
+ "ttft_max": 0.4058342077769339,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 21.34320473816755,
+ "seconds": 11.947596583,
+ "prefill_seconds": 0.405067875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.405350209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 83,
+ "drafted": 360,
+ "accepted": 172,
+ "acceptance_rate": 0.4777777777777778,
+ "tokens_per_round": 3.0843373493975905
+ },
+ "sent": 3604841.762581833,
+ "first": 3604842.168416041,
+ "last": 3604854.115861375,
+ "chunks": 84
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 41.72305834262606,
+ "ttft_max": 1.741847958881408,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.618586816004251,
+ "seconds": 24.014495,
+ "prefill_seconds": 0.4395565,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.439896125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 95,
+ "drafted": 231,
+ "accepted": 160,
+ "acceptance_rate": 0.6926406926406926,
+ "tokens_per_round": 2.694736842105263
+ },
+ "sent": 3604854.117007833,
+ "first": 3604854.557580416,
+ "last": 3604878.571969666,
+ "chunks": 96
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.85651813294774,
+ "seconds": 23.488193625,
+ "prefill_seconds": 0.872232791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.312090042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 217,
+ "accepted": 158,
+ "acceptance_rate": 0.728110599078341,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3604854.117268125,
+ "first": 3604855.429762958,
+ "last": 3604878.917895041,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.65955182687684,
+ "seconds": 23.9222065,
+ "prefill_seconds": 0.877797084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.878160459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 227,
+ "accepted": 158,
+ "acceptance_rate": 0.6960352422907489,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3604854.117089583,
+ "first": 3604854.995781625,
+ "last": 3604878.917951916,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 11.017110258099947,
+ "seconds": 23.145815375,
+ "prefill_seconds": 1.301217375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.741100167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 98,
+ "drafted": 212,
+ "accepted": 157,
+ "acceptance_rate": 0.7405660377358491,
+ "tokens_per_round": 2.6122448979591835
+ },
+ "sent": 3604854.116930541,
+ "first": 3604855.8587785,
+ "last": 3604879.004493958,
+ "chunks": 99
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 36.12399860205154,
+ "ttft_max": 0.20252579217776656,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 36.123212024726776,
+ "seconds": 7.059172917,
+ "prefill_seconds": 0.200018875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.200550916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 55,
+ "drafted": 310,
+ "accepted": 200,
+ "acceptance_rate": 0.6451612903225806,
+ "tokens_per_round": 4.654545454545454
+ },
+ "sent": 3604889.219564666,
+ "first": 3604889.422090458,
+ "last": 3604896.481109666,
+ "chunks": 56
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 49.44176860435623,
+ "ttft_max": 0.8374648750759661,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.837368778818945,
+ "seconds": 19.863883666,
+ "prefill_seconds": 0.409651375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.409944125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 82,
+ "drafted": 216,
+ "accepted": 173,
+ "acceptance_rate": 0.8009259259259259,
+ "tokens_per_round": 3.1219512195121952
+ },
+ "sent": 3604896.482677875,
+ "first": 3604896.893221833,
+ "last": 3604916.757026166,
+ "chunks": 83
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.754208356907291,
+ "seconds": 19.993400834,
+ "prefill_seconds": 0.636964958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.836840709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 84,
+ "drafted": 216,
+ "accepted": 171,
+ "acceptance_rate": 0.7916666666666666,
+ "tokens_per_round": 3.0476190476190474
+ },
+ "sent": 3604896.482735666,
+ "first": 3604897.320200541,
+ "last": 3604917.313493791,
+ "chunks": 85
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.619169315959981,
+ "seconds": 20.207352292,
+ "prefill_seconds": 0.622653792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.622908709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 84,
+ "drafted": 217,
+ "accepted": 171,
+ "acceptance_rate": 0.7880184331797235,
+ "tokens_per_round": 3.0476190476190474
+ },
+ "sent": 3604896.482804208,
+ "first": 3604897.106206625,
+ "last": 3604917.313513666,
+ "chunks": 85
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.894398186944702,
+ "seconds": 19.776029583,
+ "prefill_seconds": 0.199632,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.199933875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 81,
+ "drafted": 214,
+ "accepted": 174,
+ "acceptance_rate": 0.8130841121495327,
+ "tokens_per_round": 3.1604938271604937
+ },
+ "sent": 3604896.4826005,
+ "first": 3604896.683183708,
+ "last": 3604916.45914225,
+ "chunks": 82
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 23.096668462332214,
+ "ttft_max": 0.4583138329908252,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.096494306274323,
+ "seconds": 11.040636584,
+ "prefill_seconds": 0.457591875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.457907792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 84,
+ "drafted": 338,
+ "accepted": 171,
+ "acceptance_rate": 0.5059171597633136,
+ "tokens_per_round": 3.0476190476190474
+ },
+ "sent": 3604917.315005208,
+ "first": 3604917.773319041,
+ "last": 3604928.813872375,
+ "chunks": 85
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 39.58191601199035,
+ "ttft_max": 1.6596389156766236,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.31685269722128,
+ "seconds": 24.71684025,
+ "prefill_seconds": 0.831686959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.248863583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 99,
+ "drafted": 219,
+ "accepted": 156,
+ "acceptance_rate": 0.7123287671232876,
+ "tokens_per_round": 2.585858585858586
+ },
+ "sent": 3604928.815203625,
+ "first": 3604930.064585041,
+ "last": 3604954.781297375,
+ "chunks": 100
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.39658399943459,
+ "seconds": 24.527287041,
+ "prefill_seconds": 1.242135334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.659193041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 101,
+ "drafted": 214,
+ "accepted": 154,
+ "acceptance_rate": 0.719626168224299,
+ "tokens_per_round": 2.5346534653465347
+ },
+ "sent": 3604928.81539575,
+ "first": 3604930.475034666,
+ "last": 3604955.00224225,
+ "chunks": 102
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.146833534859063,
+ "seconds": 25.13099275,
+ "prefill_seconds": 0.83436225,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.834691042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 99,
+ "drafted": 232,
+ "accepted": 156,
+ "acceptance_rate": 0.6724137931034483,
+ "tokens_per_round": 2.585858585858586
+ },
+ "sent": 3604928.815273833,
+ "first": 3604929.650407416,
+ "last": 3604954.781352958,
+ "chunks": 100
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.981063525080605,
+ "seconds": 25.548379625,
+ "prefill_seconds": 0.416843875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.417150625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 99,
+ "drafted": 235,
+ "accepted": 156,
+ "acceptance_rate": 0.6638297872340425,
+ "tokens_per_round": 2.585858585858586
+ },
+ "sent": 3604928.815126666,
+ "first": 3604929.2328985,
+ "last": 3604954.781213166,
+ "chunks": 100
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 30.31858681923586,
+ "ttft_max": 0.19988291710615158,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 30.318100784216046,
+ "seconds": 8.410817083,
+ "prefill_seconds": 0.19909825,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.199392041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 52,
+ "drafted": 376,
+ "accepted": 203,
+ "acceptance_rate": 0.5398936170212766,
+ "tokens_per_round": 4.923076923076923
+ },
+ "sent": 3604965.120085958,
+ "first": 3604965.319968875,
+ "last": 3604973.730651125,
+ "chunks": 53
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 42.716145327090175,
+ "ttft_max": 1.1741919172927737,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.813385107043793,
+ "seconds": 23.581884625,
+ "prefill_seconds": 0.575820834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.57606975,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 242,
+ "accepted": 190,
+ "acceptance_rate": 0.7851239669421488,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 3604973.732724208,
+ "first": 3604974.309460291,
+ "last": 3604997.891232416,
+ "chunks": 66
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.952668570562086,
+ "seconds": 23.281997292,
+ "prefill_seconds": 0.875755958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.876025834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 239,
+ "accepted": 190,
+ "acceptance_rate": 0.7949790794979079,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 3604973.732836041,
+ "first": 3604974.609402333,
+ "last": 3604997.891275625,
+ "chunks": 66
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 11.094450179357176,
+ "seconds": 22.984464834,
+ "prefill_seconds": 0.894214334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.1735915,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 239,
+ "accepted": 190,
+ "acceptance_rate": 0.7949790794979079,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 3604973.732783083,
+ "first": 3604974.906975,
+ "last": 3604997.891289791,
+ "chunks": 66
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.679046785710884,
+ "seconds": 23.878535708,
+ "prefill_seconds": 0.27908075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.279380125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 244,
+ "accepted": 190,
+ "acceptance_rate": 0.7786885245901639,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 3604973.732628458,
+ "first": 3604974.012730708,
+ "last": 3604997.891204208,
+ "chunks": 66
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 23.72393278235333,
+ "ttft_max": 0.3962868331000209,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.72372098737278,
+ "seconds": 10.748735417,
+ "prefill_seconds": 0.395573708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.395857292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 469,
+ "accepted": 180,
+ "acceptance_rate": 0.3837953091684435,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3604997.893435625,
+ "first": 3604998.289722458,
+ "last": 3605009.038361916,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 37.63360072289366,
+ "ttft_max": 1.6586603340692818,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.523506182264372,
+ "seconds": 26.775852834,
+ "prefill_seconds": 0.412750042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.413028125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 95,
+ "drafted": 226,
+ "accepted": 160,
+ "acceptance_rate": 0.7079646017699115,
+ "tokens_per_round": 2.694736842105263
+ },
+ "sent": 3605009.039835666,
+ "first": 3605009.453469416,
+ "last": 3605036.229273375,
+ "chunks": 96
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.861389867154255,
+ "seconds": 25.858423958,
+ "prefill_seconds": 1.24508075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.658040666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 219,
+ "accepted": 158,
+ "acceptance_rate": 0.7214611872146118,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3605009.039891291,
+ "first": 3605010.698551625,
+ "last": 3605036.556908375,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.674276576325884,
+ "seconds": 26.358560042,
+ "prefill_seconds": 0.830139,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.830437917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 95,
+ "drafted": 225,
+ "accepted": 160,
+ "acceptance_rate": 0.7111111111111111,
+ "tokens_per_round": 2.694736842105263
+ },
+ "sent": 3605009.039945166,
+ "first": 3605009.870907166,
+ "last": 3605036.229306958,
+ "chunks": 96
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.70758935738823,
+ "seconds": 26.268107417,
+ "prefill_seconds": 1.248113042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.248405541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 219,
+ "accepted": 158,
+ "acceptance_rate": 0.7214611872146118,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3605009.039748541,
+ "first": 3605010.288857416,
+ "last": 3605036.556876958,
+ "chunks": 98
+ }
+ ]
+ }
+ ],
+ "performance-qwen-fresh-cost-records": [
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 36.15852812983743,
+ "ttft_max": 0.19951058318838477,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 36.15814210107307,
+ "seconds": 7.0523535,
+ "prefill_seconds": 0.198717,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.198999834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 54,
+ "drafted": 355,
+ "accepted": 201,
+ "acceptance_rate": 0.5661971830985916,
+ "tokens_per_round": 4.7407407407407405
+ },
+ "sent": 3606027.273496208,
+ "first": 3606027.473006791,
+ "last": 3606034.525285,
+ "chunks": 55
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 51.576538557442284,
+ "ttft_max": 0.8468245826661587,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 12.894425707530571,
+ "seconds": 19.775987375,
+ "prefill_seconds": 0.199771833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.200094417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 82,
+ "drafted": 210,
+ "accepted": 173,
+ "acceptance_rate": 0.8238095238095238,
+ "tokens_per_round": 3.1219512195121952
+ },
+ "sent": 3606034.526158833,
+ "first": 3606034.726779708,
+ "last": 3606054.502661083,
+ "chunks": 83
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 13.32983899880677,
+ "seconds": 19.1300135,
+ "prefill_seconds": 0.846160167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.846441083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 82,
+ "drafted": 210,
+ "accepted": 173,
+ "acceptance_rate": 0.8238095238095238,
+ "tokens_per_round": 3.1219512195121952
+ },
+ "sent": 3606034.526308833,
+ "first": 3606035.373133416,
+ "last": 3606054.5032135,
+ "chunks": 83
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 13.178562543878034,
+ "seconds": 19.349606541,
+ "prefill_seconds": 0.626363083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.626663041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 82,
+ "drafted": 210,
+ "accepted": 173,
+ "acceptance_rate": 0.8238095238095238,
+ "tokens_per_round": 3.1219512195121952
+ },
+ "sent": 3606034.526230625,
+ "first": 3606035.153333916,
+ "last": 3606054.503172208,
+ "chunks": 83
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 13.034482495107342,
+ "seconds": 19.563492459,
+ "prefill_seconds": 0.412400709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.412706292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 82,
+ "drafted": 210,
+ "accepted": 173,
+ "acceptance_rate": 0.8238095238095238,
+ "tokens_per_round": 3.1219512195121952
+ },
+ "sent": 3606034.526077,
+ "first": 3606034.93936725,
+ "last": 3606054.503073791,
+ "chunks": 83
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 26.04380517925345,
+ "ttft_max": 0.3833406246267259,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 26.043570663924534,
+ "seconds": 9.791284125,
+ "prefill_seconds": 0.382626125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.382920334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 448,
+ "accepted": 180,
+ "acceptance_rate": 0.4017857142857143,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606054.50418025,
+ "first": 3606054.887520875,
+ "last": 3606064.678716833,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 25.201173385079827,
+ "ttft_max": 31.02874766662717,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 25.97921266330204,
+ "seconds": 9.815539959,
+ "prefill_seconds": 0.372683292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 20.837619541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 463,
+ "accepted": 180,
+ "acceptance_rate": 0.38876889848812096,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606064.67963875,
+ "first": 3606085.517932916,
+ "last": 3606095.333423208,
+ "chunks": 76
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 25.956132195884578,
+ "seconds": 9.824268041,
+ "prefill_seconds": 0.375099959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 31.028073875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 463,
+ "accepted": 180,
+ "acceptance_rate": 0.38876889848812096,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606064.679766916,
+ "first": 3606095.708514583,
+ "last": 3606105.5327065,
+ "chunks": 76
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 25.998247972971406,
+ "seconds": 9.80835325,
+ "prefill_seconds": 0.377797958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.378097291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 463,
+ "accepted": 180,
+ "acceptance_rate": 0.38876889848812096,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606064.67970125,
+ "first": 3606065.058400625,
+ "last": 3606074.866651333,
+ "chunks": 76
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 25.735948028532654,
+ "seconds": 9.908319667,
+ "prefill_seconds": 0.370257542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 10.556647125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 463,
+ "accepted": 180,
+ "acceptance_rate": 0.38876889848812096,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606064.679569333,
+ "first": 3606075.236987916,
+ "last": 3606085.145233291,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 31.075094534881394,
+ "ttft_max": 0.20538291614502668,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 31.072944666747166,
+ "seconds": 8.206496125,
+ "prefill_seconds": 0.203727334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.204398334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 55,
+ "drafted": 365,
+ "accepted": 200,
+ "acceptance_rate": 0.547945205479452,
+ "tokens_per_round": 4.654545454545454
+ },
+ "sent": 3606141.369936875,
+ "first": 3606141.575319791,
+ "last": 3606149.781248166,
+ "chunks": 56
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 41.92490293167692,
+ "ttft_max": 0.9278107918798923,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.826532270883073,
+ "seconds": 23.553248041,
+ "prefill_seconds": 0.228371666,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.228656208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 66,
+ "drafted": 247,
+ "accepted": 189,
+ "acceptance_rate": 0.7651821862348178,
+ "tokens_per_round": 3.878787878787879
+ },
+ "sent": 3606149.7824475,
+ "first": 3606150.011670291,
+ "last": 3606173.564840375,
+ "chunks": 67
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.93183481567368,
+ "seconds": 23.326367833,
+ "prefill_seconds": 0.455304625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.455602625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 66,
+ "drafted": 245,
+ "accepted": 189,
+ "acceptance_rate": 0.7714285714285715,
+ "tokens_per_round": 3.878787878787879
+ },
+ "sent": 3606149.782563416,
+ "first": 3606150.238595125,
+ "last": 3606173.564934041,
+ "chunks": 67
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.791077036134208,
+ "seconds": 23.63063475,
+ "prefill_seconds": 0.698629208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.927254084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 69,
+ "drafted": 236,
+ "accepted": 186,
+ "acceptance_rate": 0.788135593220339,
+ "tokens_per_round": 3.710144927536232
+ },
+ "sent": 3606149.782507666,
+ "first": 3606150.710318458,
+ "last": 3606174.340885833,
+ "chunks": 70
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.683139854491918,
+ "seconds": 23.869387041,
+ "prefill_seconds": 0.68825425,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.688510666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 69,
+ "drafted": 239,
+ "accepted": 186,
+ "acceptance_rate": 0.7782426778242678,
+ "tokens_per_round": 3.710144927536232
+ },
+ "sent": 3606149.782372166,
+ "first": 3606150.471545041,
+ "last": 3606174.340864916,
+ "chunks": 70
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 21.680902825813735,
+ "ttft_max": 0.4451714172028005,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 21.680688840498675,
+ "seconds": 11.761618917,
+ "prefill_seconds": 0.444441917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.444756875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 463,
+ "accepted": 180,
+ "acceptance_rate": 0.38876889848812096,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606174.342055833,
+ "first": 3606174.78722725,
+ "last": 3606186.548730083,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 35.18775727391821,
+ "ttft_max": 1.921789416577667,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.267201928012476,
+ "seconds": 27.516396209,
+ "prefill_seconds": 1.4708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.92109,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 94,
+ "drafted": 224,
+ "accepted": 161,
+ "acceptance_rate": 0.71875,
+ "tokens_per_round": 2.723404255319149
+ },
+ "sent": 3606186.550534833,
+ "first": 3606188.47232425,
+ "last": 3606215.988622041,
+ "chunks": 95
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 8.900161211192339,
+ "seconds": 28.651166417,
+ "prefill_seconds": 0.450010667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.4503955,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 92,
+ "drafted": 237,
+ "accepted": 163,
+ "acceptance_rate": 0.6877637130801688,
+ "tokens_per_round": 2.782608695652174
+ },
+ "sent": 3606186.550593958,
+ "first": 3606187.001462875,
+ "last": 3606215.65254575,
+ "chunks": 93
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 8.942051678556043,
+ "seconds": 28.516945458,
+ "prefill_seconds": 0.47025075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.920595625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 94,
+ "drafted": 245,
+ "accepted": 161,
+ "acceptance_rate": 0.6571428571428571,
+ "tokens_per_round": 2.723404255319149
+ },
+ "sent": 3606186.550662875,
+ "first": 3606187.47172825,
+ "last": 3606215.988588166,
+ "chunks": 95
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.103197704543579,
+ "seconds": 28.012134667,
+ "prefill_seconds": 0.975065292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.425407291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 94,
+ "drafted": 234,
+ "accepted": 161,
+ "acceptance_rate": 0.688034188034188,
+ "tokens_per_round": 2.723404255319149
+ },
+ "sent": 3606186.550445166,
+ "first": 3606187.976557666,
+ "last": 3606215.988817541,
+ "chunks": 95
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 32.82984563370725,
+ "ttft_max": 0.2014766251668334,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 32.83012336049014,
+ "seconds": 7.767256833,
+ "prefill_seconds": 0.200702875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.200988792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 55,
+ "drafted": 354,
+ "accepted": 200,
+ "acceptance_rate": 0.5649717514124294,
+ "tokens_per_round": 4.654545454545454
+ },
+ "sent": 3606225.58774575,
+ "first": 3606225.789222375,
+ "last": 3606233.556544916,
+ "chunks": 56
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 39.476843897617066,
+ "ttft_max": 0.9977558329701424,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.06532199933952,
+ "seconds": 25.334509916,
+ "prefill_seconds": 0.743525709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.743825583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 64,
+ "drafted": 241,
+ "accepted": 191,
+ "acceptance_rate": 0.7925311203319502,
+ "tokens_per_round": 4.0
+ },
+ "sent": 3606233.558710833,
+ "first": 3606234.30318025,
+ "last": 3606259.637624083,
+ "chunks": 65
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.966010124797743,
+ "seconds": 25.586969791,
+ "prefill_seconds": 0.491034333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.491316208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 64,
+ "drafted": 242,
+ "accepted": 191,
+ "acceptance_rate": 0.7892561983471075,
+ "tokens_per_round": 4.0
+ },
+ "sent": 3606233.558772708,
+ "first": 3606234.050711375,
+ "last": 3606259.63764625,
+ "chunks": 65
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.869207552850355,
+ "seconds": 25.83794075,
+ "prefill_seconds": 0.240055,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.240312917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 64,
+ "drafted": 244,
+ "accepted": 191,
+ "acceptance_rate": 0.7827868852459017,
+ "tokens_per_round": 4.0
+ },
+ "sent": 3606233.558825416,
+ "first": 3606233.799724416,
+ "last": 3606259.637587541,
+ "chunks": 65
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.166927138522293,
+ "seconds": 25.081324625,
+ "prefill_seconds": 0.75667025,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.996916125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 64,
+ "drafted": 239,
+ "accepted": 191,
+ "acceptance_rate": 0.799163179916318,
+ "tokens_per_round": 4.0
+ },
+ "sent": 3606233.558633875,
+ "first": 3606234.556389708,
+ "last": 3606259.637656208,
+ "chunks": 65
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 19.40846506022387,
+ "ttft_max": 0.4472993751987815,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 19.408321894553616,
+ "seconds": 13.138693875,
+ "prefill_seconds": 0.446586166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.4468845,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 74,
+ "drafted": 471,
+ "accepted": 181,
+ "acceptance_rate": 0.3842887473460722,
+ "tokens_per_round": 3.4594594594594597
+ },
+ "sent": 3606259.639965625,
+ "first": 3606260.087265,
+ "last": 3606273.225861958,
+ "chunks": 75
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 37.247923082251035,
+ "ttft_max": 2.0259827501140535,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.487299469359217,
+ "seconds": 26.878038458,
+ "prefill_seconds": 1.035085542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.035508875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 233,
+ "accepted": 159,
+ "acceptance_rate": 0.6824034334763949,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3606273.227919,
+ "first": 3606274.264037375,
+ "last": 3606301.141950958,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.66677787410069,
+ "seconds": 26.379006875,
+ "prefill_seconds": 1.005173042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.534582791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 229,
+ "accepted": 159,
+ "acceptance_rate": 0.6943231441048034,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3606273.228045125,
+ "first": 3606274.76313025,
+ "last": 3606301.141967875,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.311984043601145,
+ "seconds": 27.384067542,
+ "prefill_seconds": 0.529000083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.529438167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 233,
+ "accepted": 159,
+ "acceptance_rate": 0.6824034334763949,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3606273.227982375,
+ "first": 3606273.757919333,
+ "last": 3606301.141920083,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.85000733950893,
+ "seconds": 25.888305583,
+ "prefill_seconds": 1.495880584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.02529125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 228,
+ "accepted": 159,
+ "acceptance_rate": 0.6973684210526315,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3606273.227833291,
+ "first": 3606275.253816041,
+ "last": 3606301.1419965,
+ "chunks": 97
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 35.350466980056616,
+ "ttft_max": 0.20126425009220839,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 35.3500488025168,
+ "seconds": 7.213568542,
+ "prefill_seconds": 0.200582458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.200864167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 54,
+ "drafted": 355,
+ "accepted": 201,
+ "acceptance_rate": 0.5661971830985916,
+ "tokens_per_round": 4.7407407407407405
+ },
+ "sent": 3606311.548035666,
+ "first": 3606311.749299916,
+ "last": 3606318.962783125,
+ "chunks": 55
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 40.423617449409754,
+ "ttft_max": 0.8631084999069571,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.975599933959169,
+ "seconds": 23.233354125,
+ "prefill_seconds": 0.207475292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.20785825,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 243,
+ "accepted": 190,
+ "acceptance_rate": 0.7818930041152263,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 3606318.964425125,
+ "first": 3606319.172912041,
+ "last": 3606342.406194541,
+ "chunks": 66
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.398315459161685,
+ "seconds": 24.523202917,
+ "prefill_seconds": 0.424516458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.424947459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 68,
+ "drafted": 244,
+ "accepted": 187,
+ "acceptance_rate": 0.7663934426229508,
+ "tokens_per_round": 3.764705882352941
+ },
+ "sent": 3606318.964586583,
+ "first": 3606319.389932416,
+ "last": 3606343.913043416,
+ "chunks": 69
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.375067787026046,
+ "seconds": 24.578152667,
+ "prefill_seconds": 0.654687291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.862650709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 70,
+ "drafted": 234,
+ "accepted": 185,
+ "acceptance_rate": 0.7905982905982906,
+ "tokens_per_round": 3.657142857142857
+ },
+ "sent": 3606318.964503083,
+ "first": 3606319.827611583,
+ "last": 3606344.405685958,
+ "chunks": 71
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.283350014062258,
+ "seconds": 24.797366583,
+ "prefill_seconds": 0.435474292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.643465125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 70,
+ "drafted": 236,
+ "accepted": 185,
+ "acceptance_rate": 0.7838983050847458,
+ "tokens_per_round": 3.657142857142857
+ },
+ "sent": 3606318.964343125,
+ "first": 3606319.608362333,
+ "last": 3606344.405664041,
+ "chunks": 71
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 23.181142415827715,
+ "ttft_max": 0.4115130412392318,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.18076293026933,
+ "seconds": 11.00050075,
+ "prefill_seconds": 0.410683709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.410999875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 468,
+ "accepted": 180,
+ "acceptance_rate": 0.38461538461538464,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606344.407623125,
+ "first": 3606344.819136166,
+ "last": 3606355.819456833,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 41.04567326689068,
+ "ttft_max": 1.7338110408745706,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.261494365126726,
+ "seconds": 24.85018175,
+ "prefill_seconds": 0.422173833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.422573417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 223,
+ "accepted": 158,
+ "acceptance_rate": 0.7085201793721974,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3606355.821066083,
+ "first": 3606356.244260083,
+ "last": 3606381.094373708,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.446103423829815,
+ "seconds": 24.411016209,
+ "prefill_seconds": 0.439312583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.861861041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 223,
+ "accepted": 158,
+ "acceptance_rate": 0.7085201793721974,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3606355.821124958,
+ "first": 3606356.683676166,
+ "last": 3606381.094626,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.64010986129906,
+ "seconds": 23.965917958,
+ "prefill_seconds": 0.884405125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.30694925,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 219,
+ "accepted": 158,
+ "acceptance_rate": 0.7214611872146118,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3606355.821194625,
+ "first": 3606357.128658333,
+ "last": 3606381.094574708,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.832718965943481,
+ "seconds": 23.5397965,
+ "prefill_seconds": 1.310551042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.7330955,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 219,
+ "accepted": 158,
+ "acceptance_rate": 0.7214611872146118,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3606355.8209675,
+ "first": 3606357.554778541,
+ "last": 3606381.094475583,
+ "chunks": 98
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 28.223626529326367,
+ "ttft_max": 0.2059693750925362,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 28.223380789937625,
+ "seconds": 9.035062167,
+ "prefill_seconds": 0.205207625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.205514375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 376,
+ "accepted": 202,
+ "acceptance_rate": 0.5372340425531915,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3606391.650607583,
+ "first": 3606391.856576958,
+ "last": 3606400.891560458,
+ "chunks": 54
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 28.761086081267262,
+ "ttft_max": 27.405207625124604,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 28.326025086672857,
+ "seconds": 9.002322042,
+ "prefill_seconds": 0.273832209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.274136208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 376,
+ "accepted": 202,
+ "acceptance_rate": 0.5372340425531915,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3606400.894081041,
+ "first": 3606401.168842916,
+ "last": 3606410.171049916,
+ "chunks": 54
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 29.82616056363124,
+ "seconds": 8.549541583,
+ "prefill_seconds": 0.265464208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 18.628984333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 368,
+ "accepted": 202,
+ "acceptance_rate": 0.5489130434782609,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3606400.893924166,
+ "first": 3606419.523642625,
+ "last": 3606428.073088041,
+ "chunks": 54
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 30.596510163274633,
+ "seconds": 8.334283833,
+ "prefill_seconds": 0.226095667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 27.404542583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 53,
+ "drafted": 368,
+ "accepted": 202,
+ "acceptance_rate": 0.5489130434782609,
+ "tokens_per_round": 4.830188679245283
+ },
+ "sent": 3606400.894007083,
+ "first": 3606428.299214708,
+ "last": 3606436.633428541,
+ "chunks": 54
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 28.817921085343936,
+ "seconds": 8.8486605,
+ "prefill_seconds": 0.238501375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 9.51489725,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 52,
+ "drafted": 377,
+ "accepted": 203,
+ "acceptance_rate": 0.5384615384615384,
+ "tokens_per_round": 4.923076923076923
+ },
+ "sent": 3606400.893823,
+ "first": 3606410.409605375,
+ "last": 3606419.258174583,
+ "chunks": 53
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 22.158009935284845,
+ "ttft_max": 0.43920179177075624,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 22.157811137460392,
+ "seconds": 11.508356959,
+ "prefill_seconds": 0.438467209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.438798917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 459,
+ "accepted": 180,
+ "acceptance_rate": 0.39215686274509803,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606436.634665083,
+ "first": 3606437.073866875,
+ "last": 3606448.582120583,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 36.93984744629104,
+ "ttft_max": 2.149744499940425,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.849580616640145,
+ "seconds": 25.889427167,
+ "prefill_seconds": 1.308605375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.149030417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 227,
+ "accepted": 159,
+ "acceptance_rate": 0.7004405286343612,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3606448.584418208,
+ "first": 3606450.734162708,
+ "last": 3606476.623452708,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.531338599247102,
+ "seconds": 26.753849666,
+ "prefill_seconds": 0.444140084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.284677042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 231,
+ "accepted": 159,
+ "acceptance_rate": 0.6883116883116883,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3606448.584539041,
+ "first": 3606449.869655166,
+ "last": 3606476.623400333,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.688840220673356,
+ "seconds": 26.318939542,
+ "prefill_seconds": 0.879069125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.71950625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 96,
+ "drafted": 227,
+ "accepted": 159,
+ "acceptance_rate": 0.7004405286343612,
+ "tokens_per_round": 2.6666666666666665
+ },
+ "sent": 3606448.58448275,
+ "first": 3606450.304553666,
+ "last": 3606476.62342825,
+ "chunks": 97
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 9.234971686120044,
+ "seconds": 27.612429,
+ "prefill_seconds": 0.425563125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.425944542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 99,
+ "drafted": 227,
+ "accepted": 156,
+ "acceptance_rate": 0.6872246696035242,
+ "tokens_per_round": 2.585858585858586
+ },
+ "sent": 3606448.584335333,
+ "first": 3606449.010994333,
+ "last": 3606476.623366333,
+ "chunks": 100
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 34.74238473493653,
+ "ttft_max": 0.20315800001844764,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 34.741093727197175,
+ "seconds": 7.340010709,
+ "prefill_seconds": 0.202288333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.202571875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 55,
+ "drafted": 355,
+ "accepted": 200,
+ "acceptance_rate": 0.5633802816901409,
+ "tokens_per_round": 4.654545454545454
+ },
+ "sent": 3606487.465374333,
+ "first": 3606487.668532333,
+ "last": 3606495.008270291,
+ "chunks": 56
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 45.67148543407574,
+ "ttft_max": 0.8624916248954833,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 11.417828712467486,
+ "seconds": 22.333493208,
+ "prefill_seconds": 0.199364125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.199731458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 70,
+ "drafted": 231,
+ "accepted": 186,
+ "acceptance_rate": 0.8051948051948052,
+ "tokens_per_round": 3.657142857142857
+ },
+ "sent": 3606495.010333583,
+ "first": 3606495.210814833,
+ "last": 3606517.544128833,
+ "chunks": 71
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 11.529024323617588,
+ "seconds": 22.118090208,
+ "prefill_seconds": 0.41482,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.415123,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 70,
+ "drafted": 227,
+ "accepted": 186,
+ "acceptance_rate": 0.8193832599118943,
+ "tokens_per_round": 3.657142857142857
+ },
+ "sent": 3606495.010474125,
+ "first": 3606495.426240125,
+ "last": 3606517.5441555,
+ "chunks": 71
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 11.766654527744414,
+ "seconds": 21.671410459,
+ "prefill_seconds": 0.662173417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.861817625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 70,
+ "drafted": 224,
+ "accepted": 186,
+ "acceptance_rate": 0.8303571428571429,
+ "tokens_per_round": 3.657142857142857
+ },
+ "sent": 3606495.010416916,
+ "first": 3606495.872908541,
+ "last": 3606517.544200166,
+ "chunks": 71
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 11.64494984033345,
+ "seconds": 21.897904542,
+ "prefill_seconds": 0.435676917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.6354045,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2b28aa2108de",
+ "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 70,
+ "drafted": 225,
+ "accepted": 186,
+ "acceptance_rate": 0.8266666666666667,
+ "tokens_per_round": 3.657142857142857
+ },
+ "sent": 3606495.010198375,
+ "first": 3606495.646404666,
+ "last": 3606517.544224625,
+ "chunks": 71
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 23.125861831112022,
+ "ttft_max": 0.43125058291479945,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 23.12566704818981,
+ "seconds": 11.026708958,
+ "prefill_seconds": 0.430470709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.430793375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 75,
+ "drafted": 448,
+ "accepted": 180,
+ "acceptance_rate": 0.4017857142857143,
+ "tokens_per_round": 3.4133333333333336
+ },
+ "sent": 3606517.5461745,
+ "first": 3606517.977425083,
+ "last": 3606529.004041166,
+ "chunks": 76
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 40.05300163872442,
+ "ttft_max": 1.7167558339424431,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.360270922105201,
+ "seconds": 24.613255958,
+ "prefill_seconds": 0.852961125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.289912375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 218,
+ "accepted": 158,
+ "acceptance_rate": 0.7247706422018348,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3606529.005617333,
+ "first": 3606530.296233,
+ "last": 3606554.909576625,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.542903217856782,
+ "seconds": 24.186886167,
+ "prefill_seconds": 1.279218041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.716137417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 217,
+ "accepted": 158,
+ "acceptance_rate": 0.728110599078341,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3606529.005735666,
+ "first": 3606530.7224915,
+ "last": 3606554.909267916,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.185256518927337,
+ "seconds": 25.036188291,
+ "prefill_seconds": 0.866474084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.866868041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 222,
+ "accepted": 158,
+ "acceptance_rate": 0.7117117117117117,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3606529.005681125,
+ "first": 3606529.873125625,
+ "last": 3606554.90949975,
+ "chunks": 98
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 10.013385130654662,
+ "seconds": 25.465913542,
+ "prefill_seconds": 0.4365775,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.436960042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a948470d6fea",
+ "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 97,
+ "drafted": 222,
+ "accepted": 158,
+ "acceptance_rate": 0.7117117117117117,
+ "tokens_per_round": 2.6391752577319587
+ },
+ "sent": 3606529.005529375,
+ "first": 3606529.443320458,
+ "last": 3606554.909149625,
+ "chunks": 98
+ }
+ ]
+ }
+ ]
+ },
+ "M5 Ultra": {
+ "functional": {
+ "resumed": {
+ "usage": {
+ "prompt_tokens": 3605,
+ "completion_tokens": 64,
+ "total_tokens": 3669,
+ "prompt_tokens_details": {
+ "cached_tokens": 3517
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 545.6868379633086,
+ "seconds": 0.115450833,
+ "prefill_seconds": 0.0341725,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.034537708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2fed3f850c53",
+ "token_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 11,
+ "drafted": 58,
+ "accepted": 52,
+ "acceptance_rate": 0.896551724137931,
+ "tokens_per_round": 5.818181818181818
+ },
+ "wall": 0.15039120900000036
+ },
+ "resume_request": {
+ "messages": [
+ {
+ "role": "user"
+ },
+ {
+ "role": "assistant"
+ },
+ {
+ "role": "user"
+ }
+ ],
+ "max_tokens": 64
+ },
+ "stress_reference_hashes": [
+ {
+ "prompt": "quicksort-raw",
+ "sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0"
+ },
+ {
+ "prompt": "rust-reverse-words",
+ "sha256": "d9499a1af105f9d0477f4fda4f2afde556c782008fe552d4a7cd0ebf71e80101"
+ },
+ {
+ "prompt": "multiply-17-23",
+ "sha256": "72bba60a60f4df7bf0b2a34dab50868cd071c0b99c2ac9d3b2f0cc99a8c6ab84"
+ },
+ {
+ "prompt": "sleep-tips",
+ "sha256": "dc9d14472ff7c82357f0686ee61236b54726924051b1222778247438e54345db"
+ },
+ {
+ "prompt": "js-debounce-raw",
+ "sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458"
+ },
+ {
+ "prompt": "linear-equation",
+ "sha256": "8ebb10c3e63f3fd3a265072c4a9ef09d88058a4fc50e488f523ce44cb0afdc0d"
+ },
+ {
+ "prompt": "fishing-village",
+ "sha256": "c1100e452897c3c18c70f946d0ae4f05a345e7f90b10bece58e0cd584daf0ade"
+ },
+ {
+ "prompt": "sql-left-join",
+ "sha256": "d04f354206e1c4c56ff5956161df55ffea050df8d858b655c26a2f8673b741b4"
+ }
+ ],
+ "stress": [
+ {
+ "slots": 6,
+ "seconds": 60,
+ "completed": 453,
+ "cancelled": 247,
+ "wrong": 0,
+ "errors": []
+ },
+ {
+ "slots": 16,
+ "seconds": 90,
+ "completed": 852,
+ "cancelled": 459,
+ "wrong": 0,
+ "errors": []
+ }
+ ],
+ "sampled": [
+ {
+ "streams": 1,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ]
+ ]
+ },
+ {
+ "streams": 4,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ]
+ ]
+ },
+ {
+ "streams": 8,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ],
+ [
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366"
+ ],
+ [
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff"
+ ],
+ [
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31"
+ ],
+ [
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf"
+ ]
+ ]
+ },
+ {
+ "streams": 16,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ],
+ [
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366"
+ ],
+ [
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff"
+ ],
+ [
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31"
+ ],
+ [
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf"
+ ],
+ [
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986",
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986",
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986"
+ ],
+ [
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351",
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351",
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351"
+ ],
+ [
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369",
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369",
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369"
+ ],
+ [
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a",
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a",
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a"
+ ],
+ [
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363",
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363",
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363"
+ ],
+ [
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398",
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398",
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398"
+ ],
+ [
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db",
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db",
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db"
+ ],
+ [
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd",
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd",
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd"
+ ]
+ ]
+ }
+ ],
+ "fresh": {
+ "usage": {
+ "prompt_tokens": 3605,
+ "completion_tokens": 64,
+ "total_tokens": 3669,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 597.1205482317299,
+ "seconds": 0.105506334,
+ "prefill_seconds": 0.308241792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.310870084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "2fed3f850c53",
+ "token_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 10,
+ "drafted": 59,
+ "accepted": 53,
+ "acceptance_rate": 0.8983050847457628,
+ "tokens_per_round": 6.4
+ },
+ "wall": 0.41713370900001223
+ },
+ "resume_equal": true
+ },
+ "long-context": [
+ {
+ "length": 4096,
+ "plain": {
+ "usage": {
+ "prompt_tokens": 4096,
+ "completion_tokens": 64,
+ "total_tokens": 4160,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 251.27072892278412,
+ "seconds": 0.250725583,
+ "prefill_seconds": 0.298777459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.299359667,
+ "sampling": "greedy",
+ "drafts": false,
+ "token_sha": "8394ececc0cc",
+ "token_sha256": "8394ececc0ccdeea4fd63598ff8309e489e7309932166b11e366f6706b8a4b3a",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 63,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "wall": 0.5511507499999997
+ },
+ "drafted": {
+ "usage": {
+ "prompt_tokens": 4096,
+ "completion_tokens": 64,
+ "total_tokens": 4160,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 313.5117331542196,
+ "seconds": 0.200949417,
+ "prefill_seconds": 0.302226708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.302388917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "8394ececc0cc",
+ "token_sha256": "8394ececc0ccdeea4fd63598ff8309e489e7309932166b11e366f6706b8a4b3a",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 26,
+ "drafted": 68,
+ "accepted": 38,
+ "acceptance_rate": 0.5588235294117647,
+ "tokens_per_round": 2.4615384615384617
+ },
+ "wall": 0.5038303750000002
+ },
+ "equal": true
+ },
+ {
+ "length": 16384,
+ "plain": {
+ "usage": {
+ "prompt_tokens": 16384,
+ "completion_tokens": 64,
+ "total_tokens": 16448,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 231.32519503146526,
+ "seconds": 0.272343875,
+ "prefill_seconds": 1.283367667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.283625625,
+ "sampling": "greedy",
+ "drafts": false,
+ "token_sha": "a2dcb347e183",
+ "token_sha256": "a2dcb347e183d9e8a0be485abb99d222ff060dfc83781384863e7bf16cf22da3",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 63,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "wall": 1.5566373330000003
+ },
+ "drafted": {
+ "usage": {
+ "prompt_tokens": 16384,
+ "completion_tokens": 64,
+ "total_tokens": 16448,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 259.7309798859134,
+ "seconds": 0.242558666,
+ "prefill_seconds": 1.342481375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 1.34274125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a2dcb347e183",
+ "token_sha256": "a2dcb347e183d9e8a0be485abb99d222ff060dfc83781384863e7bf16cf22da3",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 32,
+ "drafted": 57,
+ "accepted": 31,
+ "acceptance_rate": 0.543859649122807,
+ "tokens_per_round": 2.0
+ },
+ "wall": 1.5860129589999996
+ },
+ "equal": true
+ },
+ {
+ "length": 65536,
+ "plain": {
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 64,
+ "total_tokens": 65600,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 157.88251974236329,
+ "seconds": 0.399030875,
+ "prefill_seconds": 7.318780875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.319519292,
+ "sampling": "greedy",
+ "drafts": false,
+ "token_sha": "c2ba0e4604c6",
+ "token_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 63,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "wall": 7.720276332999999
+ },
+ "drafted": {
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 64,
+ "total_tokens": 65600,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 151.86195081442588,
+ "seconds": 0.414850459,
+ "prefill_seconds": 7.556797125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.557524875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "c2ba0e4604c6",
+ "token_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 36,
+ "drafted": 122,
+ "accepted": 27,
+ "acceptance_rate": 0.22131147540983606,
+ "tokens_per_round": 1.7777777777777777
+ },
+ "wall": 7.974062624999998
+ },
+ "equal": true
+ },
+ {
+ "length": 130816,
+ "plain": {
+ "usage": {
+ "prompt_tokens": 130816,
+ "completion_tokens": 64,
+ "total_tokens": 130880,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 116.19828729405333,
+ "seconds": 0.542176666,
+ "prefill_seconds": 20.563776416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 20.565105666,
+ "sampling": "greedy",
+ "drafts": false,
+ "token_sha": "3cc988610216",
+ "token_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 63,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "wall": 21.110205083000004
+ },
+ "drafted": {
+ "usage": {
+ "prompt_tokens": 130816,
+ "completion_tokens": 64,
+ "total_tokens": 130880,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 230.23142702247748,
+ "seconds": 0.273637708,
+ "prefill_seconds": 21.036062792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 21.037378792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3cc988610216",
+ "token_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "min_rows": 5
+ },
+ "speculative": {
+ "rounds": 12,
+ "drafted": 108,
+ "accepted": 51,
+ "acceptance_rate": 0.4722222222222222,
+ "tokens_per_round": 5.333333333333333
+ },
+ "wall": 21.314060583999996
+ },
+ "equal": true
+ }
+ ],
+ "performance-lfm2": [
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 289.9714374495252,
+ "ttft_max": 0.016745999999997707,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 289.9559952775242,
+ "seconds": 0.879443792,
+ "prefill_seconds": 0.016395833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.016501792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 124.128642,
+ "first": 124.145388,
+ "last": 125.024784958,
+ "chunks": 217
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 652.2080388699854,
+ "ttft_max": 0.06611995800000159,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 166.96049804643943,
+ "seconds": 1.527307375,
+ "prefill_seconds": 0.01635325,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049324708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 125.0255105,
+ "first": 125.075392583,
+ "last": 126.602660541,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 168.34147176327625,
+ "seconds": 1.51477825,
+ "prefill_seconds": 0.01636675,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0656875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 125.025627875,
+ "first": 125.091747833,
+ "last": 126.606481416,
+ "chunks": 225
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 163.45264548507336,
+ "seconds": 1.560084875,
+ "prefill_seconds": 0.016379542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.016536916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 125.025571583,
+ "first": 125.04256325,
+ "last": 126.602603041,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 165.19216464411534,
+ "seconds": 1.54365675,
+ "prefill_seconds": 0.016455834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.032961709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 125.025436875,
+ "first": 125.059020333,
+ "last": 126.602634583,
+ "chunks": 224
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 830.9301245588248,
+ "ttft_max": 0.13169079099999692,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.19389804837425,
+ "seconds": 2.447360208,
+ "prefill_seconds": 0.016465708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.016684167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 126.607762625,
+ "first": 126.625097166,
+ "last": 129.0724045,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.89724081177302,
+ "seconds": 2.4309505,
+ "prefill_seconds": 0.016419292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033021917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 126.607816791,
+ "first": 126.641500916,
+ "last": 129.072416125,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.5459214960778,
+ "seconds": 2.349236125,
+ "prefill_seconds": 0.016382917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.114713417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 126.608144291,
+ "first": 126.7233725,
+ "last": 129.072579958,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.79394994721177,
+ "seconds": 2.365624417,
+ "prefill_seconds": 0.016353916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098507667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 126.607865791,
+ "first": 126.70698675,
+ "last": 129.072642708,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.05374686342388,
+ "seconds": 2.381981084,
+ "prefill_seconds": 0.016367334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082089875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 126.608013291,
+ "first": 126.690629583,
+ "last": 129.072869166,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.60819370489229,
+ "seconds": 2.414585375,
+ "prefill_seconds": 0.0163805,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049418458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 126.607912666,
+ "first": 126.657892958,
+ "last": 129.072431375,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.95296996983765,
+ "seconds": 2.340459375,
+ "prefill_seconds": 0.016380709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131116792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ },
+ "sent": 126.60806725,
+ "first": 126.739758041,
+ "last": 129.08017725,
+ "chunks": 237
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.3282934211501,
+ "seconds": 2.398232792,
+ "prefill_seconds": 0.016363417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.065817125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 126.607685583,
+ "first": 126.674259291,
+ "last": 129.072453583,
+ "chunks": 235
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 828.3957621161079,
+ "ttft_max": 0.26417679200000066,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.4837114578651,
+ "seconds": 4.858650292,
+ "prefill_seconds": 0.016387708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082543416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082258416,
+ "first": 129.166236416,
+ "last": 134.025300916,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.132941351539856,
+ "seconds": 4.891341125,
+ "prefill_seconds": 0.016411458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049863042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082833125,
+ "first": 129.133453208,
+ "last": 134.024820291,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.0191678536274,
+ "seconds": 4.809581333,
+ "prefill_seconds": 0.016347333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131707875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082449458,
+ "first": 129.215360125,
+ "last": 134.025603666,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.66131800556297,
+ "seconds": 4.842263917,
+ "prefill_seconds": 0.016385584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098904375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082311083,
+ "first": 129.182623833,
+ "last": 134.025126708,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.78590461813402,
+ "seconds": 4.924119833,
+ "prefill_seconds": 0.016984208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0171665,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082358166,
+ "first": 129.100618625,
+ "last": 134.024723791,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.83961379629883,
+ "seconds": 4.82592475,
+ "prefill_seconds": 0.016392042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.115322833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.0824055,
+ "first": 129.199020583,
+ "last": 134.025367041,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.93609398148886,
+ "seconds": 4.727817333,
+ "prefill_seconds": 0.016383958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.212811334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082654708,
+ "first": 129.297255958,
+ "last": 134.025415541,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.56612729160864,
+ "seconds": 4.760471083,
+ "prefill_seconds": 0.016407792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.180273792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082776333,
+ "first": 129.264506041,
+ "last": 134.025465958,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.30800705704185,
+ "seconds": 4.874970666,
+ "prefill_seconds": 0.016384042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0662055,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082710958,
+ "first": 129.1498505,
+ "last": 134.0255115,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.959096956934374,
+ "seconds": 4.907706541,
+ "prefill_seconds": 0.016405084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033492417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082581166,
+ "first": 129.117039416,
+ "last": 134.024706166,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.38055057810182,
+ "seconds": 4.777020792,
+ "prefill_seconds": 0.016359792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.164066458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082493833,
+ "first": 129.248097666,
+ "last": 134.025556375,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.750243175321984,
+ "seconds": 4.744164583,
+ "prefill_seconds": 0.016359167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.196493458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.083095208,
+ "first": 129.2808835,
+ "last": 134.025245041,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.12412620540813,
+ "seconds": 4.711392458,
+ "prefill_seconds": 0.0163735,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.229135875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082913458,
+ "first": 129.313639791,
+ "last": 134.025751041,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.19990749800366,
+ "seconds": 4.793241417,
+ "prefill_seconds": 0.016356667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.147830375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082537041,
+ "first": 129.231729,
+ "last": 134.025192166,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.31266282813846,
+ "seconds": 4.695037708,
+ "prefill_seconds": 0.016368,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.245525542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.083007458,
+ "first": 129.330005458,
+ "last": 134.025707458,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.502041390310886,
+ "seconds": 4.678723833,
+ "prefill_seconds": 0.016355791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.261876959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 20,
+ "accepted": 16,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 129.082185333,
+ "first": 129.346362125,
+ "last": 134.02580075,
+ "chunks": 240
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 261.8548695057814,
+ "ttft_max": 0.019914957999986882,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 261.84384531549165,
+ "seconds": 0.973862875,
+ "prefill_seconds": 0.019466083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019607666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ },
+ "sent": 134.028428208,
+ "first": 134.048343166,
+ "last": 135.022165041,
+ "chunks": 248
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 623.8913057741527,
+ "ttft_max": 0.07612033399999518,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 155.97151868320893,
+ "seconds": 1.634913875,
+ "prefill_seconds": 0.018871917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019013209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 135.024678125,
+ "first": 135.044165708,
+ "last": 136.679035208,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 159.655644687928,
+ "seconds": 1.5971875,
+ "prefill_seconds": 0.018885958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.056715125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 135.024729791,
+ "first": 135.081912541,
+ "last": 136.679057833,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 157.79077236147523,
+ "seconds": 1.616064084,
+ "prefill_seconds": 0.018858833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.037831417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 135.024784708,
+ "first": 135.06302475,
+ "last": 136.679047833,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 161.55734099774537,
+ "seconds": 1.578386958,
+ "prefill_seconds": 0.0188095,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.075517208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 135.024608416,
+ "first": 135.10072875,
+ "last": 136.679065875,
+ "chunks": 249
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 804.7903304448168,
+ "ttft_max": 0.15244674999999575,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.45497930497726,
+ "seconds": 2.4180935,
+ "prefill_seconds": 0.018895542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.132696,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 136.682299791,
+ "first": 136.816032333,
+ "last": 139.234285625,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.50799246585228,
+ "seconds": 2.512117458,
+ "prefill_seconds": 0.018809333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038682083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 136.682349875,
+ "first": 136.721842625,
+ "last": 139.233927375,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.2725854447154,
+ "seconds": 2.493336791,
+ "prefill_seconds": 0.018816833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057453708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 136.682500916,
+ "first": 136.740661625,
+ "last": 139.233946833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.75387442870228,
+ "seconds": 2.530920041,
+ "prefill_seconds": 0.019629541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019855542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 136.682447458,
+ "first": 136.70303025,
+ "last": 139.23390925,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.1153164506986,
+ "seconds": 2.403046125,
+ "prefill_seconds": 0.018809541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.151517333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 136.682397375,
+ "first": 136.834844125,
+ "last": 139.237851958,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.04935504488036,
+ "seconds": 2.474542416,
+ "prefill_seconds": 0.018781333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076166791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 136.682617583,
+ "first": 136.759446666,
+ "last": 139.234074125,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.6375463110905,
+ "seconds": 2.43698375,
+ "prefill_seconds": 0.018829917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113811833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 136.682559875,
+ "first": 136.797134958,
+ "last": 139.234247458,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.83638983320756,
+ "seconds": 2.455786458,
+ "prefill_seconds": 0.018852292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094927917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 136.682232541,
+ "first": 136.778290083,
+ "last": 139.234064083,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 798.7138944430546,
+ "ttft_max": 0.30219858300000624,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.038958238356024,
+ "seconds": 4.900174958,
+ "prefill_seconds": 0.018812083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.225735791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.241876291,
+ "first": 139.469755916,
+ "last": 144.370635541,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.645081133940984,
+ "seconds": 4.843757375,
+ "prefill_seconds": 0.018843625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.282080375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.241926416,
+ "first": 139.526226541,
+ "last": 144.370152958,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.114745571121645,
+ "seconds": 5.08832275,
+ "prefill_seconds": 0.018857834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038370542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.242242916,
+ "first": 139.281345041,
+ "last": 144.3696365,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.250450280175464,
+ "seconds": 4.975566041,
+ "prefill_seconds": 0.018787792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.150630333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.242420083,
+ "first": 139.394317125,
+ "last": 144.370054333,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.44518779537533,
+ "seconds": 4.956731833,
+ "prefill_seconds": 0.01885675,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169326625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.241972833,
+ "first": 139.413183916,
+ "last": 144.370551041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.67643014217816,
+ "seconds": 5.031925084,
+ "prefill_seconds": 0.018838334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.09432575,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.2420175,
+ "first": 139.337810041,
+ "last": 144.37069625,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.48755177359346,
+ "seconds": 5.050749958,
+ "prefill_seconds": 0.018804167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.07552175,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.242474,
+ "first": 139.318966,
+ "last": 144.369743,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.92971047107042,
+ "seconds": 5.107179625,
+ "prefill_seconds": 0.019435333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019633792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.242060333,
+ "first": 139.262484125,
+ "last": 144.369621125,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.64220612545368,
+ "seconds": 4.937821583,
+ "prefill_seconds": 0.018907209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.188229709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.242191541,
+ "first": 139.432086583,
+ "last": 144.369915875,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.058301350286555,
+ "seconds": 4.994290708,
+ "prefill_seconds": 0.018909833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.13187375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.242326416,
+ "first": 139.375525041,
+ "last": 144.370594041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.840043719916856,
+ "seconds": 4.918977333,
+ "prefill_seconds": 0.0188405,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.207001916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.24274875,
+ "first": 139.450944,
+ "last": 144.370281958,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.300370060335396,
+ "seconds": 5.069545208,
+ "prefill_seconds": 0.018811042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.056869542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.24210275,
+ "first": 139.300158791,
+ "last": 144.369966,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.8499599998342,
+ "seconds": 4.82498,
+ "prefill_seconds": 0.018821833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.300614791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.242850333,
+ "first": 139.545048916,
+ "last": 144.370439416,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.44091554383292,
+ "seconds": 4.862615333,
+ "prefill_seconds": 0.018820458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.263346125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.2421485,
+ "first": 139.507378208,
+ "last": 144.370390166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.23848569029113,
+ "seconds": 4.8814585,
+ "prefill_seconds": 0.018789917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.24454025,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.242551333,
+ "first": 139.488551541,
+ "last": 144.3701935,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.86513617793781,
+ "seconds": 5.013257,
+ "prefill_seconds": 0.018808041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113095333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 139.241802291,
+ "first": 139.35662025,
+ "last": 144.369869333,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 289.72587991488484,
+ "ttft_max": 0.017593750000003183,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 289.6930751741181,
+ "seconds": 0.880241959,
+ "prefill_seconds": 0.016860375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017127334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 147.223546083,
+ "first": 147.241139833,
+ "last": 148.121282125,
+ "chunks": 217
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 649.6021903057296,
+ "ttft_max": 0.06667979200000218,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 162.79983373680727,
+ "seconds": 1.566340666,
+ "prefill_seconds": 0.017153125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017348375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 148.12660075,
+ "first": 148.144440958,
+ "last": 149.710751083,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 167.6232283395662,
+ "seconds": 1.521268875,
+ "prefill_seconds": 0.016347458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066252792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 148.126719708,
+ "first": 148.1933995,
+ "last": 149.714632708,
+ "chunks": 225
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 164.51003069848784,
+ "seconds": 1.550057458,
+ "prefill_seconds": 0.0162855,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0336135,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 148.126664666,
+ "first": 148.16073,
+ "last": 149.710764916,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 166.2580215102581,
+ "seconds": 1.533760583,
+ "prefill_seconds": 0.0163095,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049878458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 148.126504958,
+ "first": 148.177041083,
+ "last": 149.710775,
+ "chunks": 224
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 829.8565857754016,
+ "ttft_max": 0.13205166699998472,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.7570662804786,
+ "seconds": 2.434203334,
+ "prefill_seconds": 0.016336958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033825708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 149.719728875,
+ "first": 149.754335333,
+ "last": 152.188516791,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.46240456557089,
+ "seconds": 2.41792325,
+ "prefill_seconds": 0.016293041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.050100792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 149.719781708,
+ "first": 149.770638708,
+ "last": 152.188533875,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.05825140106275,
+ "seconds": 2.4505505,
+ "prefill_seconds": 0.017265125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017462875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 149.719938416,
+ "first": 149.737996958,
+ "last": 152.188544291,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.17673990755674,
+ "seconds": 2.401655958,
+ "prefill_seconds": 0.016282125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066412625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 149.719830541,
+ "first": 149.786910916,
+ "last": 152.188621416,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.78403340480372,
+ "seconds": 2.344094,
+ "prefill_seconds": 0.016315416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131539833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ },
+ "sent": 149.720132166,
+ "first": 149.852183833,
+ "last": 152.196253041,
+ "chunks": 237
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.38048753710204,
+ "seconds": 2.352822042,
+ "prefill_seconds": 0.016322125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.115258875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 149.720000333,
+ "first": 149.835876291,
+ "last": 152.189067625,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.90075464692875,
+ "seconds": 2.385390083,
+ "prefill_seconds": 0.016310875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082632,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 149.719877166,
+ "first": 149.8032345,
+ "last": 152.18876775,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.63435531635936,
+ "seconds": 2.369132042,
+ "prefill_seconds": 0.016312375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098949667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 149.719657166,
+ "first": 149.819553125,
+ "last": 152.188947875,
+ "chunks": 235
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 828.6755735521535,
+ "ttft_max": 0.26237383299999806,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.976483494829445,
+ "seconds": 4.906064875,
+ "prefill_seconds": 0.016308208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033549875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.20229475,
+ "first": 152.237289541,
+ "last": 157.143305666,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.506561542949406,
+ "seconds": 4.678335833,
+ "prefill_seconds": 0.016273417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.261174,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202958583,
+ "first": 152.465332416,
+ "last": 157.143935,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.80460451006316,
+ "seconds": 4.922342375,
+ "prefill_seconds": 0.017116209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017279959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202357791,
+ "first": 152.220975375,
+ "last": 157.143273958,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.57511701564012,
+ "seconds": 4.759672292,
+ "prefill_seconds": 0.016269667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.179849709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202687208,
+ "first": 152.38391825,
+ "last": 157.144389083,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.31631183691749,
+ "seconds": 4.694722292,
+ "prefill_seconds": 0.016261083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.244935209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202408916,
+ "first": 152.44905775,
+ "last": 157.144159333,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.497494527736826,
+ "seconds": 4.857374667,
+ "prefill_seconds": 0.016310375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082430375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202455875,
+ "first": 152.286180208,
+ "last": 157.144008375,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.323544098349856,
+ "seconds": 4.873523084,
+ "prefill_seconds": 0.016281834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066054958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202730708,
+ "first": 152.269866833,
+ "last": 157.144050166,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.392395322641754,
+ "seconds": 4.775961042,
+ "prefill_seconds": 0.016313,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.163628,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202636541,
+ "first": 152.367643291,
+ "last": 157.1442535,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.75897038755569,
+ "seconds": 4.743394417,
+ "prefill_seconds": 0.016290917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.196119792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202501708,
+ "first": 152.400212541,
+ "last": 157.144343833,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.2098882511113,
+ "seconds": 4.792342333,
+ "prefill_seconds": 0.01627575,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.147333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202592625,
+ "first": 152.351327666,
+ "last": 157.144213208,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.85089150476571,
+ "seconds": 4.824894959,
+ "prefill_seconds": 0.016256834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.114773875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202779791,
+ "first": 152.318726541,
+ "last": 157.143661333,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.129992851779946,
+ "seconds": 4.710881834,
+ "prefill_seconds": 0.016289166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.228667375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202546791,
+ "first": 152.432780375,
+ "last": 157.144299583,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.0301257905778,
+ "seconds": 4.8085875,
+ "prefill_seconds": 0.016313,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131061416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202824,
+ "first": 152.335047458,
+ "last": 157.144494458,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.149293060798676,
+ "seconds": 4.889807417,
+ "prefill_seconds": 0.016292833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.04980925,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202868833,
+ "first": 152.253582125,
+ "last": 157.143424291,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.67394686960922,
+ "seconds": 4.841102958,
+ "prefill_seconds": 0.016283458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098661333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.20291025,
+ "first": 152.302458083,
+ "last": 157.143725,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.94291697444283,
+ "seconds": 4.727219333,
+ "prefill_seconds": 0.016285708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.212410958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 152.202220541,
+ "first": 152.416500125,
+ "last": 157.143870666,
+ "chunks": 240
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 261.4389358917785,
+ "ttft_max": 0.01995562499999437,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 261.42741066011195,
+ "seconds": 0.975414167,
+ "prefill_seconds": 0.019505167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019642667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ },
+ "sent": 157.151165291,
+ "first": 157.171120916,
+ "last": 158.146492083,
+ "chunks": 248
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 621.9575399660765,
+ "ttft_max": 0.07646687499999416,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 159.14182827765322,
+ "seconds": 1.602344291,
+ "prefill_seconds": 0.018828959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057355417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 158.15290625,
+ "first": 158.210693208,
+ "last": 159.812987583,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 155.48832633280173,
+ "seconds": 1.6399945,
+ "prefill_seconds": 0.019593875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0197315,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 158.152959,
+ "first": 158.173025416,
+ "last": 159.812993458,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 161.03016363348462,
+ "seconds": 1.58355425,
+ "prefill_seconds": 0.018808125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076165958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 158.153034708,
+ "first": 158.229501583,
+ "last": 159.81300875,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 157.29485342441458,
+ "seconds": 1.621159208,
+ "prefill_seconds": 0.018828042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038547125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 158.152837458,
+ "first": 158.191863208,
+ "last": 159.812975875,
+ "chunks": 249
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 803.6481953347434,
+ "ttft_max": 0.15192016599999647,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.61238418108967,
+ "seconds": 2.53447925,
+ "prefill_seconds": 0.019770458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01991025,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 159.820251541,
+ "first": 159.840839125,
+ "last": 162.375272125,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.36327475042472,
+ "seconds": 2.515704042,
+ "prefill_seconds": 0.018790042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038690417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 159.820299,
+ "first": 159.859620833,
+ "last": 162.375289666,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.12461416990364,
+ "seconds": 2.496949458,
+ "prefill_seconds": 0.018780041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057416125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 159.820439166,
+ "first": 159.878424416,
+ "last": 162.375407166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.47681621037697,
+ "seconds": 2.440732875,
+ "prefill_seconds": 0.018787542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113625458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 159.820388875,
+ "first": 159.93475425,
+ "last": 162.3757295,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.93935860945503,
+ "seconds": 2.407037416,
+ "prefill_seconds": 0.018755042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.151151375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 159.8203455,
+ "first": 159.972265666,
+ "last": 162.379263291,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.28566012405747,
+ "seconds": 2.42198225,
+ "prefill_seconds": 0.018747875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.132366375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 159.82058875,
+ "first": 159.953500375,
+ "last": 162.375637625,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.68018231006161,
+ "seconds": 2.459486416,
+ "prefill_seconds": 0.018772292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094867541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 159.820525666,
+ "first": 159.915963208,
+ "last": 162.375684708,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.89784024999332,
+ "seconds": 2.478186125,
+ "prefill_seconds": 0.018772834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076230041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 159.82018225,
+ "first": 159.897197333,
+ "last": 162.37579275,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 798.2060644244926,
+ "ttft_max": 0.30242166599998654,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.082243715282694,
+ "seconds": 5.091624917,
+ "prefill_seconds": 0.018790083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038452791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387256333,
+ "first": 162.426992083,
+ "last": 167.518563958,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.99798519348889,
+ "seconds": 4.904036167,
+ "prefill_seconds": 0.018809625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.225475875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387305541,
+ "first": 162.614833375,
+ "last": 167.519646833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.21441254901633,
+ "seconds": 4.979067167,
+ "prefill_seconds": 0.018818959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.150812333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387613125,
+ "first": 162.539740208,
+ "last": 167.51956775,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.39927234860693,
+ "seconds": 4.866479792,
+ "prefill_seconds": 0.018790792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.26276875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387708291,
+ "first": 162.652404833,
+ "last": 167.519421208,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.64125397401233,
+ "seconds": 5.035420334,
+ "prefill_seconds": 0.018788459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387351458,
+ "first": 162.483348291,
+ "last": 167.519237833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.800133781219394,
+ "seconds": 4.922767209,
+ "prefill_seconds": 0.018728084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2068905,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387667333,
+ "first": 162.59601925,
+ "last": 167.519464416,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.603122245333005,
+ "seconds": 4.941561458,
+ "prefill_seconds": 0.0187735,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.188214166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387397375,
+ "first": 162.577278791,
+ "last": 167.519096166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.02185605900568,
+ "seconds": 4.997858167,
+ "prefill_seconds": 0.018797625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131972417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387799041,
+ "first": 162.520916083,
+ "last": 167.519039375,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.453946602893055,
+ "seconds": 5.054114042,
+ "prefill_seconds": 0.018778125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.075802416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387568333,
+ "first": 162.46453425,
+ "last": 167.519609166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.197727318076026,
+ "seconds": 4.885270166,
+ "prefill_seconds": 0.018782167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2441205,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387998958,
+ "first": 162.633622416,
+ "last": 167.519282833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.80526185126259,
+ "seconds": 4.829064208,
+ "prefill_seconds": 0.018736541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.300208875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387441125,
+ "first": 162.689862791,
+ "last": 167.51918725,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.40736524587146,
+ "seconds": 4.960378708,
+ "prefill_seconds": 0.018771959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169516292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387843791,
+ "first": 162.55851125,
+ "last": 167.519140791,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.26736818885785,
+ "seconds": 5.0728735,
+ "prefill_seconds": 0.018771375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057141208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387483916,
+ "first": 162.445761958,
+ "last": 167.518660041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.830627266884015,
+ "seconds": 5.0166605,
+ "prefill_seconds": 0.018771666,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113197791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.3877535,
+ "first": 162.502124708,
+ "last": 167.519378458,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.60158047773111,
+ "seconds": 4.847763084,
+ "prefill_seconds": 0.01871975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.281475459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387527541,
+ "first": 162.671121,
+ "last": 167.519336791,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.89828698936166,
+ "seconds": 5.110395875,
+ "prefill_seconds": 0.01938875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019613167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 162.387183083,
+ "first": 162.408184791,
+ "last": 167.518552875,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 345.7276309402865,
+ "ttft_max": 0.019197541000011142,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 345.65026843416723,
+ "seconds": 0.737739916,
+ "prefill_seconds": 0.01819825,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.018542042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 306,
+ "accepted": 166,
+ "acceptance_rate": 0.5424836601307189,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 170.434448875,
+ "first": 170.453646416,
+ "last": 171.19122125,
+ "chunks": 90
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 633.5514893351391,
+ "ttft_max": 0.0739344999999787,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 161.98510734078445,
+ "seconds": 1.574218792,
+ "prefill_seconds": 0.0179905,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.054373459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 171.217551833,
+ "first": 171.273054125,
+ "last": 172.84715575,
+ "chunks": 225
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 160.54720369974092,
+ "seconds": 1.588317916,
+ "prefill_seconds": 0.017848833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.036408458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 41,
+ "accepted": 32,
+ "acceptance_rate": 0.7804878048780488,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 171.217838833,
+ "first": 171.255051541,
+ "last": 172.843258958,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 158.761572570732,
+ "seconds": 1.606182125,
+ "prefill_seconds": 0.018322083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.018604875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 171.217703708,
+ "first": 171.237184,
+ "last": 172.843268166,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 164.94290112406395,
+ "seconds": 1.545989541,
+ "prefill_seconds": 0.018218917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.072570042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 222,
+ "drafted": 41,
+ "accepted": 33,
+ "acceptance_rate": 0.8048780487804879,
+ "tokens_per_round": 1.1531531531531531
+ },
+ "sent": 171.217352083,
+ "first": 171.291286583,
+ "last": 172.83718675,
+ "chunks": 223
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 812.1249994900322,
+ "ttft_max": 0.13950241699998855,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.18747313821937,
+ "seconds": 2.4954135,
+ "prefill_seconds": 0.017388125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017560833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 172.856445875,
+ "first": 172.874706416,
+ "last": 175.370184833,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.15704404096843,
+ "seconds": 2.424944542,
+ "prefill_seconds": 0.017013917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.087911917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 172.856716333,
+ "first": 172.945285125,
+ "last": 175.370540791,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.70128001877501,
+ "seconds": 2.458986041,
+ "prefill_seconds": 0.017051292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.054009708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 172.856507541,
+ "first": 172.91115875,
+ "last": 175.370447958,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.67751940439814,
+ "seconds": 2.390381792,
+ "prefill_seconds": 0.01699375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.138761167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 24,
+ "accepted": 18,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 172.856785416,
+ "first": 172.996287833,
+ "last": 175.386635,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.98779025304378,
+ "seconds": 2.476021666,
+ "prefill_seconds": 0.019396792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.036931875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 172.856606041,
+ "first": 172.89410975,
+ "last": 175.370159291,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.43588215244323,
+ "seconds": 2.395808583,
+ "prefill_seconds": 0.017016833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.121752083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 28,
+ "accepted": 21,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 172.856832125,
+ "first": 172.979291833,
+ "last": 175.375062791,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.42807278452388,
+ "seconds": 2.441872125,
+ "prefill_seconds": 0.017100292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.071068333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 172.856558916,
+ "first": 172.928262041,
+ "last": 175.370236125,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.89930597631496,
+ "seconds": 2.407947792,
+ "prefill_seconds": 0.016990542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.104894583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 172.856355,
+ "first": 172.962278375,
+ "last": 175.370590625,
+ "chunks": 234
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 794.6944987638544,
+ "ttft_max": 0.27339062500001887,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.011193681879966,
+ "seconds": 5.0988585,
+ "prefill_seconds": 0.017067916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.051142125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396549583,
+ "first": 175.449037666,
+ "last": 180.54798075,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.34534065594796,
+ "seconds": 5.065016875,
+ "prefill_seconds": 0.016973375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.085040708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396940291,
+ "first": 175.483047708,
+ "last": 180.548542,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.096281589595456,
+ "seconds": 4.894783125,
+ "prefill_seconds": 0.017031459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.254624417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396599125,
+ "first": 175.653388375,
+ "last": 180.548447583,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.91615750589105,
+ "seconds": 4.911765667,
+ "prefill_seconds": 0.017022167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2376825,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.397019541,
+ "first": 175.636356583,
+ "last": 180.54875,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.55892191507427,
+ "seconds": 4.94579775,
+ "prefill_seconds": 0.017085083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.203777167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396647041,
+ "first": 175.602322416,
+ "last": 180.548898041,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.859778745725706,
+ "seconds": 5.013785083,
+ "prefill_seconds": 0.017029166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.136140291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396692166,
+ "first": 175.534216125,
+ "last": 180.548619,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.51630928871303,
+ "seconds": 5.047874708,
+ "prefill_seconds": 0.017060833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.102066541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.397157708,
+ "first": 175.500112791,
+ "last": 180.548385375,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.38126740894436,
+ "seconds": 4.962898209,
+ "prefill_seconds": 0.0169445,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.186799916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.397239166,
+ "first": 175.585225166,
+ "last": 180.548285625,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.27741474109795,
+ "seconds": 4.877823459,
+ "prefill_seconds": 0.016977833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.271606333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396979666,
+ "first": 175.670370291,
+ "last": 180.548828166,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.03211749181326,
+ "seconds": 4.996853208,
+ "prefill_seconds": 0.016988583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.152903666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396747,
+ "first": 175.551213166,
+ "last": 180.548215,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.844554216135364,
+ "seconds": 5.115904917,
+ "prefill_seconds": 0.01705125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.034082291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396788666,
+ "first": 175.431957333,
+ "last": 180.547865708,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.68731040786727,
+ "seconds": 5.030844958,
+ "prefill_seconds": 0.017073458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119139792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396873458,
+ "first": 175.517188,
+ "last": 180.548184416,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.67913488754507,
+ "seconds": 5.132939625,
+ "prefill_seconds": 0.017062709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017221458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.397074583,
+ "first": 175.41490575,
+ "last": 180.547812625,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.20732437437765,
+ "seconds": 4.979756375,
+ "prefill_seconds": 0.017061458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169880459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396829125,
+ "first": 175.568279208,
+ "last": 180.548662125,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.73702453243566,
+ "seconds": 4.928772041,
+ "prefill_seconds": 0.017009958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.220804,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.397466333,
+ "first": 175.619330208,
+ "last": 180.548715958,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.17800811704017,
+ "seconds": 5.081907584,
+ "prefill_seconds": 0.017031125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.068205584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 175.396474958,
+ "first": 175.466075916,
+ "last": 180.548954125,
+ "chunks": 253
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 356.283045357237,
+ "ttft_max": 0.0198536249999961,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 356.2623676742896,
+ "seconds": 0.715764625,
+ "prefill_seconds": 0.019405291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019556458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ },
+ "sent": 180.559402541,
+ "first": 180.579256166,
+ "last": 181.29497925,
+ "chunks": 92
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 617.4407646534572,
+ "ttft_max": 0.07852779099999907,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 154.376093024738,
+ "seconds": 1.651810167,
+ "prefill_seconds": 0.019667583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019799375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 181.305110583,
+ "first": 181.325383083,
+ "last": 182.977196416,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 158.08517319160634,
+ "seconds": 1.6130545,
+ "prefill_seconds": 0.019354208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.058541542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 181.305161208,
+ "first": 181.364150666,
+ "last": 182.977168083,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 156.21058841312555,
+ "seconds": 1.63241175,
+ "prefill_seconds": 0.019416625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.039194125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 181.305215083,
+ "first": 181.344788708,
+ "last": 182.977363416,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 160.00901903699273,
+ "seconds": 1.593660167,
+ "prefill_seconds": 0.019406792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0779435,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 181.305036,
+ "first": 181.383563791,
+ "last": 182.977279,
+ "chunks": 250
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 793.7909080606516,
+ "ttft_max": 0.15909400000001028,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 99.37856571901393,
+ "seconds": 2.565945666,
+ "prefill_seconds": 0.019888792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.020052083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 182.988145125,
+ "first": 183.008811875,
+ "last": 185.574721083,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.20306644084567,
+ "seconds": 2.447144875,
+ "prefill_seconds": 0.019436125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.138857125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 182.988194666,
+ "first": 183.127767625,
+ "last": 185.575160208,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.99723881470433,
+ "seconds": 2.5248215,
+ "prefill_seconds": 0.019431667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.061271291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 182.988337291,
+ "first": 183.050054916,
+ "last": 185.5750405,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.5767566060795,
+ "seconds": 2.485943292,
+ "prefill_seconds": 0.019458459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.100055583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 182.988463333,
+ "first": 183.0889355,
+ "last": 185.575254833,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.7821996501377,
+ "seconds": 2.505349667,
+ "prefill_seconds": 0.019409708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.080589292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 182.988240625,
+ "first": 183.069476125,
+ "last": 185.574807333,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.2295879774243,
+ "seconds": 2.544158917,
+ "prefill_seconds": 0.02180525,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.041865,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 182.988285916,
+ "first": 183.030620833,
+ "last": 185.574762083,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.38082320911384,
+ "seconds": 2.466608333,
+ "prefill_seconds": 0.019389542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119433209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 182.988398041,
+ "first": 183.108331958,
+ "last": 185.575021041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.86764061650318,
+ "seconds": 2.431636666,
+ "prefill_seconds": 0.019388958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.158255167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 6,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 182.988072916,
+ "first": 183.147166916,
+ "last": 185.578758166,
+ "chunks": 256
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 789.3346380570592,
+ "ttft_max": 0.3117172080000046,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.52943356426058,
+ "seconds": 5.148453791,
+ "prefill_seconds": 0.019454542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.039253959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.590695791,
+ "first": 185.631209958,
+ "last": 190.779631666,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.870230469382534,
+ "seconds": 5.012754958,
+ "prefill_seconds": 0.019362375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.17457875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.590743708,
+ "first": 185.767068708,
+ "last": 190.779988041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.904543704315714,
+ "seconds": 5.109755166,
+ "prefill_seconds": 0.019406417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.077998375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.591052625,
+ "first": 185.670025708,
+ "last": 190.780126208,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.479579513021015,
+ "seconds": 5.051547625,
+ "prefill_seconds": 0.01936975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.135929542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.590789041,
+ "first": 185.728268083,
+ "last": 190.780533541,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.09559499773575,
+ "seconds": 5.090267917,
+ "prefill_seconds": 0.01942475,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.097430375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.5912585,
+ "first": 185.689452666,
+ "last": 190.780443958,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.71594163114495,
+ "seconds": 5.1291395,
+ "prefill_seconds": 0.019402875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.05859625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.591108541,
+ "first": 185.65061025,
+ "last": 190.77987325,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.285914114393485,
+ "seconds": 4.877030541,
+ "prefill_seconds": 0.019357291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.309896666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.591211708,
+ "first": 185.902928916,
+ "last": 190.780663958,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.07942152629365,
+ "seconds": 4.89636775,
+ "prefill_seconds": 0.019388875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.290604208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.590877541,
+ "first": 185.88357275,
+ "last": 190.7804955,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.67053368265316,
+ "seconds": 4.9351145,
+ "prefill_seconds": 0.019424083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.251939834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.591358833,
+ "first": 185.844793958,
+ "last": 190.78015475,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.34316665682364,
+ "seconds": 5.167888834,
+ "prefill_seconds": 0.01960475,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019844209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.590833791,
+ "first": 185.611753583,
+ "last": 190.779611083,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.67403630212287,
+ "seconds": 5.032162792,
+ "prefill_seconds": 0.019429083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.155263208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.591302958,
+ "first": 185.747701375,
+ "last": 190.780263916,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.26752849289349,
+ "seconds": 4.973908583,
+ "prefill_seconds": 0.019454792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.213334541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.591161458,
+ "first": 185.80593725,
+ "last": 190.780202541,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.28697180415821,
+ "seconds": 5.070895917,
+ "prefill_seconds": 0.019438792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.116738916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.590919041,
+ "first": 185.708885791,
+ "last": 190.780083375,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.87409404691879,
+ "seconds": 4.9157485,
+ "prefill_seconds": 0.019384709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.271264208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.591003125,
+ "first": 185.864176083,
+ "last": 190.780342958,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.06723171619991,
+ "seconds": 4.993417333,
+ "prefill_seconds": 0.019408791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.193930083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.590959916,
+ "first": 185.786477416,
+ "last": 190.7806215,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.46810585048602,
+ "seconds": 4.954524667,
+ "prefill_seconds": 0.019427209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.232649,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 185.5906235,
+ "first": 185.825365833,
+ "last": 190.780392083,
+ "chunks": 255
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 289.5446835251194,
+ "ttft_max": 0.01664883299997655,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 289.52884856931587,
+ "seconds": 0.88074125,
+ "prefill_seconds": 0.016308417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.016413334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 192.945268875,
+ "first": 192.961917708,
+ "last": 193.842610791,
+ "chunks": 217
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 649.3601917375305,
+ "ttft_max": 0.06672287500001062,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 162.73396034928217,
+ "seconds": 1.566974708,
+ "prefill_seconds": 0.017142042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017301542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 193.855370208,
+ "first": 193.873108583,
+ "last": 195.440038791,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 164.45010581596875,
+ "seconds": 1.550622292,
+ "prefill_seconds": 0.016355,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033608292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 193.85548125,
+ "first": 193.8894655,
+ "last": 195.440049791,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 167.56919930261643,
+ "seconds": 1.521759375,
+ "prefill_seconds": 0.016328708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066281583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 193.855425916,
+ "first": 193.922148791,
+ "last": 195.4438855,
+ "chunks": 225
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 166.20392994080697,
+ "seconds": 1.53425975,
+ "prefill_seconds": 0.01636925,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.04996875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 193.855305916,
+ "first": 193.905832083,
+ "last": 195.440065833,
+ "chunks": 224
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 830.0258741687314,
+ "ttft_max": 0.1323993340000129,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.77697039105169,
+ "seconds": 2.433740917,
+ "prefill_seconds": 0.016276833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033837958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 195.456540916,
+ "first": 195.490997,
+ "last": 197.9247045,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.08126446027792,
+ "seconds": 2.450008667,
+ "prefill_seconds": 0.017435542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017601792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 195.456586416,
+ "first": 195.474721708,
+ "last": 197.924690666,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.2004508521493,
+ "seconds": 2.40111975,
+ "prefill_seconds": 0.016355792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066483375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 195.456443291,
+ "first": 195.523651708,
+ "last": 197.924731416,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.80566369748101,
+ "seconds": 2.343628,
+ "prefill_seconds": 0.016292792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131531875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ },
+ "sent": 195.456493416,
+ "first": 195.58889275,
+ "last": 197.932476416,
+ "chunks": 237
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.40519500956965,
+ "seconds": 2.352285792,
+ "prefill_seconds": 0.016349209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.115261166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 195.456750041,
+ "first": 195.572604125,
+ "last": 197.92507575,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.92444446704627,
+ "seconds": 2.384861584,
+ "prefill_seconds": 0.016282792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082664833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 195.456802458,
+ "first": 195.53992875,
+ "last": 197.925125416,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.48317214793134,
+ "seconds": 2.417447208,
+ "prefill_seconds": 0.016295375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.050117875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 195.456674125,
+ "first": 195.50729325,
+ "last": 197.924719583,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.65957033632907,
+ "seconds": 2.368577166,
+ "prefill_seconds": 0.016295583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098914542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 195.456369541,
+ "first": 195.55624475,
+ "last": 197.924863041,
+ "chunks": 235
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 831.5239140286818,
+ "ttft_max": 0.26237274999999727,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.768413276464976,
+ "seconds": 4.742561375,
+ "prefill_seconds": 0.016265,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.179626458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.945928541,
+ "first": 198.127401375,
+ "last": 202.870412958,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.04038814956047,
+ "seconds": 4.807657125,
+ "prefill_seconds": 0.016352458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.114574583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946512666,
+ "first": 198.062228083,
+ "last": 202.870853291,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.15677557025132,
+ "seconds": 4.889105916,
+ "prefill_seconds": 0.016377709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033394542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.94597725,
+ "first": 197.980583041,
+ "last": 202.869653833,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.707620700272,
+ "seconds": 4.661142209,
+ "prefill_seconds": 0.016277583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.260805917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946568125,
+ "first": 198.208940875,
+ "last": 202.870685583,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.95393048432777,
+ "seconds": 4.726254375,
+ "prefill_seconds": 0.016303875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.195915292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946023791,
+ "first": 198.143698791,
+ "last": 202.870647958,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.861242596841535,
+ "seconds": 4.823950166,
+ "prefill_seconds": 0.01634625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098226958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946314666,
+ "first": 198.045870291,
+ "last": 202.870816041,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.98265410181364,
+ "seconds": 4.9054825,
+ "prefill_seconds": 0.016940541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017084917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946113333,
+ "first": 197.964199458,
+ "last": 202.869641791,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.327417121993896,
+ "seconds": 4.693762625,
+ "prefill_seconds": 0.016323084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.228316708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946240291,
+ "first": 198.176346041,
+ "last": 202.87076675,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.221206566966686,
+ "seconds": 4.791323167,
+ "prefill_seconds": 0.016321584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.130839125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946069333,
+ "first": 198.078556458,
+ "last": 202.870587708,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.14007566234809,
+ "seconds": 4.7100045,
+ "prefill_seconds": 0.016314625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.212103375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946365916,
+ "first": 198.160016541,
+ "last": 202.87048725,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.506466084817106,
+ "seconds": 4.856544708,
+ "prefill_seconds": 0.016302458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.065666708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.94646675,
+ "first": 198.013189166,
+ "last": 202.869760958,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.401544906694156,
+ "seconds": 4.77514275,
+ "prefill_seconds": 0.01628175,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.147076541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946155583,
+ "first": 198.094844833,
+ "last": 202.870727166,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.68279198187996,
+ "seconds": 4.840290167,
+ "prefill_seconds": 0.016327667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.081921709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946608208,
+ "first": 198.029519625,
+ "last": 202.8702055,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.517634027718074,
+ "seconds": 4.677385667,
+ "prefill_seconds": 0.016312833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.244603667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946196291,
+ "first": 198.192671041,
+ "last": 202.870100875,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.33092992421609,
+ "seconds": 4.87283525,
+ "prefill_seconds": 0.016299958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049342125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.946422875,
+ "first": 197.9968835,
+ "last": 202.869671625,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.5834534300851,
+ "seconds": 4.758931791,
+ "prefill_seconds": 0.016287667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.163394917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 17,
+ "accepted": 16,
+ "acceptance_rate": 0.9411764705882353,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 197.945856791,
+ "first": 198.111126625,
+ "last": 202.870542458,
+ "chunks": 240
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 261.0430596423368,
+ "ttft_max": 0.019998125000000755,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 261.0291981241995,
+ "seconds": 0.976902208,
+ "prefill_seconds": 0.019545625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0196965,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ },
+ "sent": 202.884711916,
+ "first": 202.904710041,
+ "last": 203.881560375,
+ "chunks": 248
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 620.6793785851976,
+ "ttft_max": 0.07659712499997795,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 160.68386640173455,
+ "seconds": 1.586967041,
+ "prefill_seconds": 0.018783417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076120167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 203.895535375,
+ "first": 203.9721325,
+ "last": 205.559074833,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 158.80574759701076,
+ "seconds": 1.605735333,
+ "prefill_seconds": 0.018791208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057380833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 203.895588625,
+ "first": 203.953345291,
+ "last": 205.559063166,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 155.16896340819196,
+ "seconds": 1.643369875,
+ "prefill_seconds": 0.019636334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019772292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 203.895639375,
+ "first": 203.915714291,
+ "last": 205.559055458,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 156.96882729897197,
+ "seconds": 1.624526375,
+ "prefill_seconds": 0.018837042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038587417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 203.895463125,
+ "first": 203.934550916,
+ "last": 205.559046166,
+ "chunks": 249
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 803.2640477075901,
+ "ttft_max": 0.15217400000000225,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.31499581811627,
+ "seconds": 2.516902833,
+ "prefill_seconds": 0.018763333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038594,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 205.573844833,
+ "first": 205.613133958,
+ "last": 208.13001625,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.23827830592693,
+ "seconds": 2.423072708,
+ "prefill_seconds": 0.018833083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.13241375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 205.573894208,
+ "first": 205.707139458,
+ "last": 208.130425166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.63429028995851,
+ "seconds": 2.460575542,
+ "prefill_seconds": 0.018792917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.09493575,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 205.573939083,
+ "first": 205.669533,
+ "last": 208.130074166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.42832582906196,
+ "seconds": 2.441866208,
+ "prefill_seconds": 0.018770416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113580292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 205.57424025,
+ "first": 205.688305958,
+ "last": 208.130318666,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.5653613853155,
+ "seconds": 2.535664333,
+ "prefill_seconds": 0.019734875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019905292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 205.574035875,
+ "first": 205.594377,
+ "last": 208.129995791,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.84966510914845,
+ "seconds": 2.479346916,
+ "prefill_seconds": 0.018774792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076161583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 205.573984791,
+ "first": 205.650730083,
+ "last": 208.130127583,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.07667828584013,
+ "seconds": 2.498122042,
+ "prefill_seconds": 0.018817875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057407125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 205.574106125,
+ "first": 205.631965333,
+ "last": 208.130049125,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.89272971052287,
+ "seconds": 2.408097333,
+ "prefill_seconds": 0.018806458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.151230584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 205.573776875,
+ "first": 205.725950875,
+ "last": 208.134015125,
+ "chunks": 253
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 799.8179957692911,
+ "ttft_max": 0.30238558299998886,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.7115797791297,
+ "seconds": 4.93119725,
+ "prefill_seconds": 0.018804875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.188510916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.149626,
+ "first": 208.340005708,
+ "last": 213.271452875,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.1077184684266,
+ "seconds": 4.893708792,
+ "prefill_seconds": 0.018815375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.225882042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.150235666,
+ "first": 208.377575416,
+ "last": 213.271850708,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.369354695341165,
+ "seconds": 5.062602083,
+ "prefill_seconds": 0.018815208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057438583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.149681458,
+ "first": 208.208409458,
+ "last": 213.2711705,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.3088641120886,
+ "seconds": 4.874890792,
+ "prefill_seconds": 0.0187635,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.244614333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.150010291,
+ "first": 208.396334166,
+ "last": 213.271311916,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.90841015481174,
+ "seconds": 4.91249875,
+ "prefill_seconds": 0.018757916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2071505,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.14973025,
+ "first": 208.35875975,
+ "last": 213.271764833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.18287023633437,
+ "seconds": 5.081415208,
+ "prefill_seconds": 0.018837875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038693083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.149776708,
+ "first": 208.189589125,
+ "last": 213.27097,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.92015856856854,
+ "seconds": 4.818579666,
+ "prefill_seconds": 0.018806625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.300774125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.150295125,
+ "first": 208.452680708,
+ "last": 213.271397083,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.745412377113176,
+ "seconds": 5.025084792,
+ "prefill_seconds": 0.018868375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094998125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.150162333,
+ "first": 208.246045458,
+ "last": 213.271578916,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.556738316460006,
+ "seconds": 5.043838042,
+ "prefill_seconds": 0.018774458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.07632475,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.150066833,
+ "first": 208.227183375,
+ "last": 213.270984583,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.99778033383685,
+ "seconds": 5.100226416,
+ "prefill_seconds": 0.019620083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019882083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.149950208,
+ "first": 208.170753625,
+ "last": 213.27092925,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.319368125751765,
+ "seconds": 4.96888425,
+ "prefill_seconds": 0.018726959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.151211916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.149820791,
+ "first": 208.302334666,
+ "last": 213.271914166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.12634641303452,
+ "seconds": 4.987643708,
+ "prefill_seconds": 0.01880325,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.132510542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.149907666,
+ "first": 208.283601583,
+ "last": 213.271712291,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.510550441416676,
+ "seconds": 4.856166958,
+ "prefill_seconds": 0.018748667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.263278042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.149864916,
+ "first": 208.415094041,
+ "last": 213.271539125,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.514815451228166,
+ "seconds": 4.950032292,
+ "prefill_seconds": 0.018847875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169946291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.150120375,
+ "first": 208.321178375,
+ "last": 213.271666708,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.713996927962626,
+ "seconds": 4.837424875,
+ "prefill_seconds": 0.018780041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.282029875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.150433958,
+ "first": 208.433877958,
+ "last": 213.271623166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.93547239679412,
+ "seconds": 5.00633425,
+ "prefill_seconds": 0.018738667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113766375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 208.149541958,
+ "first": 208.264785541,
+ "last": 213.271209083,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 356.51506556960055,
+ "ttft_max": 0.0180762499999787,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 356.4531651992678,
+ "seconds": 0.7153815,
+ "prefill_seconds": 0.0173945,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017618917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 298,
+ "accepted": 166,
+ "acceptance_rate": 0.5570469798657718,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 216.2409795,
+ "first": 216.25905575,
+ "last": 216.974313041,
+ "chunks": 90
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 645.4558312923031,
+ "ttft_max": 0.0689538329999948,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 163.5263986917888,
+ "seconds": 1.55938125,
+ "prefill_seconds": 0.017043042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.034629166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 41,
+ "accepted": 32,
+ "acceptance_rate": 0.7804878048780488,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 216.991414125,
+ "first": 217.026535,
+ "last": 218.585877125,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 161.7575112179627,
+ "seconds": 1.57643375,
+ "prefill_seconds": 0.017402209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017555875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 216.991469916,
+ "first": 217.009495541,
+ "last": 218.585886,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 167.8327708426794,
+ "seconds": 1.519369541,
+ "prefill_seconds": 0.016975375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.068522416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 222,
+ "drafted": 40,
+ "accepted": 33,
+ "acceptance_rate": 0.825,
+ "tokens_per_round": 1.1531531531531531
+ },
+ "sent": 216.991527583,
+ "first": 217.060481416,
+ "last": 218.57982675,
+ "chunks": 223
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 164.90833606031902,
+ "seconds": 1.546313583,
+ "prefill_seconds": 0.016955791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.051565416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 216.991326166,
+ "first": 217.04349525,
+ "last": 218.589774083,
+ "chunks": 225
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 812.625302340236,
+ "ttft_max": 0.1390859169999885,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.95701780856994,
+ "seconds": 2.40663625,
+ "prefill_seconds": 0.0169995,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.10443525,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 218.606616583,
+ "first": 218.712053958,
+ "last": 221.118782208,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.25043516050957,
+ "seconds": 2.493876917,
+ "prefill_seconds": 0.017096458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017283167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 218.606668125,
+ "first": 218.624661916,
+ "last": 221.1184945,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.47972886410659,
+ "seconds": 2.440664833,
+ "prefill_seconds": 0.017086292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.070542625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 218.606761375,
+ "first": 218.678019125,
+ "last": 221.118684333,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.75777254356842,
+ "seconds": 2.457647208,
+ "prefill_seconds": 0.016906459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.053488041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 218.606812291,
+ "first": 218.660929833,
+ "last": 221.118551541,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.04917456095748,
+ "seconds": 2.47454675,
+ "prefill_seconds": 0.019356792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.036653375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 218.606879208,
+ "first": 218.644024416,
+ "last": 221.118540875,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.21514330611976,
+ "seconds": 2.4236055,
+ "prefill_seconds": 0.017024,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.087507292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 218.606715416,
+ "first": 218.695048,
+ "last": 221.118941916,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.74145728307404,
+ "seconds": 2.388949959,
+ "prefill_seconds": 0.017080458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.138532334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 24,
+ "accepted": 18,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 218.607053583,
+ "first": 218.7461395,
+ "last": 221.135044,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.49642829772976,
+ "seconds": 2.3944465,
+ "prefill_seconds": 0.016995209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.121442542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 28,
+ "accepted": 21,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 218.606543458,
+ "first": 218.729046666,
+ "last": 221.123450958,
+ "chunks": 235
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 796.1534147494743,
+ "ttft_max": 0.27384479200000555,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.26800940189429,
+ "seconds": 5.072808791,
+ "prefill_seconds": 0.016912166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.06778725,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.152768583,
+ "first": 221.222254416,
+ "last": 226.295372291,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.37578979471439,
+ "seconds": 4.868661666,
+ "prefill_seconds": 0.017050542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.271505791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.152817833,
+ "first": 221.426662625,
+ "last": 226.295984708,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.30002477833443,
+ "seconds": 4.970757833,
+ "prefill_seconds": 0.01705575,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169690292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.152910875,
+ "first": 221.324447916,
+ "last": 226.295665875,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.12491136915193,
+ "seconds": 4.987783708,
+ "prefill_seconds": 0.016988417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.152699916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.152996875,
+ "first": 221.307384041,
+ "last": 226.295858625,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.77814942225688,
+ "seconds": 5.021845083,
+ "prefill_seconds": 0.017025625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.118747375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.153127833,
+ "first": 221.273364333,
+ "last": 226.295630291,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.93494626963064,
+ "seconds": 5.106644125,
+ "prefill_seconds": 0.017024709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.034276375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.152865333,
+ "first": 221.188368333,
+ "last": 226.295026208,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.01157661760684,
+ "seconds": 4.902754667,
+ "prefill_seconds": 0.017041375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.237577584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.153558125,
+ "first": 221.392554083,
+ "last": 226.295546625,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.76917431684454,
+ "seconds": 5.123653416,
+ "prefill_seconds": 0.01709925,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017298625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.15331425,
+ "first": 221.17134425,
+ "last": 226.294957958,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.95109289172138,
+ "seconds": 5.004799417,
+ "prefill_seconds": 0.0170235,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.135711625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.15339475,
+ "first": 221.290385,
+ "last": 226.295898458,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.60725271892516,
+ "seconds": 5.038803458,
+ "prefill_seconds": 0.01699775,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.101790334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.153193625,
+ "first": 221.256347375,
+ "last": 226.295453666,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.10126144743879,
+ "seconds": 5.089692208,
+ "prefill_seconds": 0.016966458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.05106175,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.152953625,
+ "first": 221.205344125,
+ "last": 226.294994416,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.65335031479425,
+ "seconds": 4.93675625,
+ "prefill_seconds": 0.017052166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.203563667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.153439041,
+ "first": 221.358512458,
+ "last": 226.295934875,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.831596943059246,
+ "seconds": 4.919778958,
+ "prefill_seconds": 0.017001833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.220550333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.153038416,
+ "first": 221.375508458,
+ "last": 226.295724916,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.47540885983634,
+ "seconds": 4.95382175,
+ "prefill_seconds": 0.017001875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.186600666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.153079916,
+ "first": 221.341449,
+ "last": 226.29581125,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.437527003547316,
+ "seconds": 5.055759375,
+ "prefill_seconds": 0.017075375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.084843917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.153274916,
+ "first": 221.239331875,
+ "last": 226.295589208,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.19304069563798,
+ "seconds": 4.885708834,
+ "prefill_seconds": 0.017049458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.254538875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 221.152695791,
+ "first": 221.409601916,
+ "last": 226.295761541,
+ "chunks": 253
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 356.0100491221774,
+ "ttft_max": 0.020198124999978972,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 355.9871871193013,
+ "seconds": 0.716317916,
+ "prefill_seconds": 0.019727208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01988125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ },
+ "sent": 226.314053625,
+ "first": 226.33425175,
+ "last": 227.050523666,
+ "chunks": 92
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 618.4685158218351,
+ "ttft_max": 0.07836916599998744,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 160.27251778546173,
+ "seconds": 1.591040083,
+ "prefill_seconds": 0.019357709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.077917208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 227.06915325,
+ "first": 227.147522416,
+ "last": 228.738690791,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 158.34876392877254,
+ "seconds": 1.610369375,
+ "prefill_seconds": 0.019305875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.058574625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 227.069204916,
+ "first": 227.128161625,
+ "last": 228.738491125,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 154.63508038201635,
+ "seconds": 1.649043667,
+ "prefill_seconds": 0.019737,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019870542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 227.069281375,
+ "first": 227.089455666,
+ "last": 228.738462583,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 156.47366856257813,
+ "seconds": 1.629667166,
+ "prefill_seconds": 0.019395083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.039290583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 227.069078708,
+ "first": 227.108854375,
+ "last": 228.738507458,
+ "chunks": 250
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 794.2858917207209,
+ "ttft_max": 0.15848308399998245,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.0587355415921,
+ "seconds": 2.523285084,
+ "prefill_seconds": 0.019405292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.061060667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 228.757089041,
+ "first": 228.818985875,
+ "last": 231.342461,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.44137453891526,
+ "seconds": 2.465164458,
+ "prefill_seconds": 0.019400916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119085792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 228.757139041,
+ "first": 228.877125083,
+ "last": 231.342672666,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.83898746886234,
+ "seconds": 2.503952625,
+ "prefill_seconds": 0.019355083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.08043925,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 228.757230458,
+ "first": 228.838339333,
+ "last": 231.342510416,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.29101108382824,
+ "seconds": 2.54260075,
+ "prefill_seconds": 0.021773917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.041741625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 228.757292416,
+ "first": 228.799579583,
+ "last": 231.342144,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.26003787647788,
+ "seconds": 2.445807667,
+ "prefill_seconds": 0.019376375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.138402208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 3,
+ "accepted": 1,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 228.757185333,
+ "first": 228.896502166,
+ "last": 231.342379708,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.63346743176672,
+ "seconds": 2.484569667,
+ "prefill_seconds": 0.019376791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.099703125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 228.757397791,
+ "first": 228.857733291,
+ "last": 231.342557458,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.92297573491601,
+ "seconds": 2.43035425,
+ "prefill_seconds": 0.019320458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.157714333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 6,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 228.757346791,
+ "first": 228.915829875,
+ "last": 231.34614375,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 99.43941680009635,
+ "seconds": 2.564375458,
+ "prefill_seconds": 0.019845875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.020029833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 228.757016125,
+ "first": 228.777799,
+ "last": 231.342134333,
+ "chunks": 255
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 789.771383093008,
+ "ttft_max": 0.31042429199999333,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.5585013758026,
+ "seconds": 5.145434041,
+ "prefill_seconds": 0.019434375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03891575,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366020625,
+ "first": 231.406101041,
+ "last": 236.551513208,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.37207354814985,
+ "seconds": 5.164863083,
+ "prefill_seconds": 0.019504125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019657625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366069541,
+ "first": 231.386665333,
+ "last": 236.551489041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.314192230717474,
+ "seconds": 4.874394292,
+ "prefill_seconds": 0.019387959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.309051,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.367078958,
+ "first": 231.67750325,
+ "last": 236.552717291,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.10618255422472,
+ "seconds": 4.893853042,
+ "prefill_seconds": 0.019403834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.289837834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366399625,
+ "first": 231.658110041,
+ "last": 236.552315041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.123972476181336,
+ "seconds": 5.087386083,
+ "prefill_seconds": 0.019403625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.09677175,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366507041,
+ "first": 231.464250583,
+ "last": 236.552484833,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.745306071496096,
+ "seconds": 5.126111791,
+ "prefill_seconds": 0.019392042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.058307,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366205541,
+ "first": 231.425497333,
+ "last": 236.552256583,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.898896549288594,
+ "seconds": 5.009931792,
+ "prefill_seconds": 0.019383541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.174048792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366115833,
+ "first": 231.541814125,
+ "last": 236.551894041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.2947249526529,
+ "seconds": 4.971271417,
+ "prefill_seconds": 0.019353375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.212683375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366611583,
+ "first": 231.580536958,
+ "last": 236.552053875,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.31405424947672,
+ "seconds": 5.068166416,
+ "prefill_seconds": 0.019370708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.1160415,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366161416,
+ "first": 231.483617208,
+ "last": 236.552527041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.09530423884891,
+ "seconds": 4.990673875,
+ "prefill_seconds": 0.01935975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.193347583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366554291,
+ "first": 231.561176666,
+ "last": 236.552657791,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.901169968812894,
+ "seconds": 4.913184041,
+ "prefill_seconds": 0.0193665,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.27052375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366248708,
+ "first": 231.638703708,
+ "last": 236.552156041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.495257347179034,
+ "seconds": 4.951912334,
+ "prefill_seconds": 0.019391792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.231982292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366697958,
+ "first": 231.59992825,
+ "last": 236.552439875,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.69738391616706,
+ "seconds": 4.932551334,
+ "prefill_seconds": 0.019402542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.251379375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366291166,
+ "first": 231.619342458,
+ "last": 236.552565583,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.93336384761851,
+ "seconds": 5.106805958,
+ "prefill_seconds": 0.019342875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.077485708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366456791,
+ "first": 231.444845416,
+ "last": 236.551823916,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.70264840649132,
+ "seconds": 5.029323083,
+ "prefill_seconds": 0.019391417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.154691708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.366335916,
+ "first": 231.522416166,
+ "last": 236.55221075,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.50744466196342,
+ "seconds": 5.048760667,
+ "prefill_seconds": 0.019405917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.135417708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 231.36594425,
+ "first": 231.50303225,
+ "last": 236.552390583,
+ "chunks": 255
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 290.8510658622243,
+ "ttft_max": 0.01669720799998231,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 290.8404504890468,
+ "seconds": 0.876769375,
+ "prefill_seconds": 0.016364292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01647625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 239.119373958,
+ "first": 239.136071166,
+ "last": 240.012808541,
+ "chunks": 217
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 651.741970635724,
+ "ttft_max": 0.06697054099998923,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 168.2030078750511,
+ "seconds": 1.516025208,
+ "prefill_seconds": 0.016288875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066519584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 240.033585875,
+ "first": 240.100556416,
+ "last": 241.616544208,
+ "chunks": 225
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 165.06071405913494,
+ "seconds": 1.544886083,
+ "prefill_seconds": 0.016394459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033796,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 240.033701625,
+ "first": 240.067897041,
+ "last": 241.612781625,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 163.32746514404067,
+ "seconds": 1.561280583,
+ "prefill_seconds": 0.017268041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01742925,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 240.033644458,
+ "first": 240.051507666,
+ "last": 241.612744708,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 166.82790179935552,
+ "seconds": 1.528521292,
+ "prefill_seconds": 0.016369292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.05020025,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 240.033513,
+ "first": 240.084273625,
+ "last": 241.6128545,
+ "chunks": 224
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 830.8281925407863,
+ "ttft_max": 0.13214558399999987,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.58861198447467,
+ "seconds": 2.415033167,
+ "prefill_seconds": 0.016334791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.050086541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 241.637057125,
+ "first": 241.687804708,
+ "last": 244.102791208,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.92960785183546,
+ "seconds": 2.340961333,
+ "prefill_seconds": 0.01633,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131459333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ },
+ "sent": 241.637417541,
+ "first": 241.769563125,
+ "last": 244.110488041,
+ "chunks": 237
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.77019400800891,
+ "seconds": 2.366145875,
+ "prefill_seconds": 0.016328375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098779375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 241.637356291,
+ "first": 241.736815583,
+ "last": 244.103246708,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.30634603982891,
+ "seconds": 2.398727917,
+ "prefill_seconds": 0.01633,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066277916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 241.637215333,
+ "first": 241.704144916,
+ "last": 244.103204708,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.03474678681029,
+ "seconds": 2.382403917,
+ "prefill_seconds": 0.016336375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082566,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 241.63729,
+ "first": 241.720486041,
+ "last": 244.103041875,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.52098436526774,
+ "seconds": 2.349775958,
+ "prefill_seconds": 0.016411959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.11517475,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 241.6371155,
+ "first": 241.753231291,
+ "last": 244.102989125,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.1788219264758,
+ "seconds": 2.447714375,
+ "prefill_seconds": 0.0172515,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0174245,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 241.637166333,
+ "first": 241.65510675,
+ "last": 244.102778208,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.87980361772534,
+ "seconds": 2.431354667,
+ "prefill_seconds": 0.016368875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033824042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 241.636985416,
+ "first": 241.67147825,
+ "last": 244.102807333,
+ "chunks": 235
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 830.8764623507997,
+ "ttft_max": 0.2633158749999893,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.2876866740977,
+ "seconds": 4.876865209,
+ "prefill_seconds": 0.016316958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0502095,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132334416,
+ "first": 244.183997083,
+ "last": 249.060898041,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.0965041857833,
+ "seconds": 4.713798125,
+ "prefill_seconds": 0.016320625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.213210958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132382875,
+ "first": 244.347323291,
+ "last": 249.061706125,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.91079714136565,
+ "seconds": 4.730035791,
+ "prefill_seconds": 0.016310959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.196879916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.1327245,
+ "first": 244.331000041,
+ "last": 249.061462708,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.46293821308609,
+ "seconds": 4.860574125,
+ "prefill_seconds": 0.016295833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066502208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.13243375,
+ "first": 244.20029975,
+ "last": 249.061247041,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.11353376479628,
+ "seconds": 4.893162708,
+ "prefill_seconds": 0.016336459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033903458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.1327845,
+ "first": 244.167669583,
+ "last": 249.060805666,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.357004782800345,
+ "seconds": 4.779128833,
+ "prefill_seconds": 0.016427583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.148012458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132483083,
+ "first": 244.282021833,
+ "last": 249.061749333,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.94034400629201,
+ "seconds": 4.909478458,
+ "prefill_seconds": 0.017405792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017608125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.13298025,
+ "first": 244.1513485,
+ "last": 249.060786416,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.540147133077845,
+ "seconds": 4.762781084,
+ "prefill_seconds": 0.016341667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.164288708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132836833,
+ "first": 244.298358583,
+ "last": 249.061825791,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.639884162991194,
+ "seconds": 4.844235584,
+ "prefill_seconds": 0.016357167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082845584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132530791,
+ "first": 244.216666208,
+ "last": 249.060937625,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.99553494433352,
+ "seconds": 4.811726125,
+ "prefill_seconds": 0.016302583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.115342583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132880583,
+ "first": 244.249265375,
+ "last": 249.061494625,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.815375363532894,
+ "seconds": 4.8281395,
+ "prefill_seconds": 0.016290834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0989905,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132577625,
+ "first": 244.232958875,
+ "last": 249.061538416,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.66295212256913,
+ "seconds": 4.664951125,
+ "prefill_seconds": 0.01633075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.262054,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132931541,
+ "first": 244.396247416,
+ "last": 249.061641625,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.28344066227193,
+ "seconds": 4.697565167,
+ "prefill_seconds": 0.01627325,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.229426625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132624416,
+ "first": 244.363604333,
+ "last": 249.061416666,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.47267817352149,
+ "seconds": 4.681245875,
+ "prefill_seconds": 0.016305875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2457435,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.133083666,
+ "first": 244.379900916,
+ "last": 249.061602625,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.17579718838869,
+ "seconds": 4.795414709,
+ "prefill_seconds": 0.01632025,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.1315775,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.13266925,
+ "first": 244.265597541,
+ "last": 249.061794458,
+ "chunks": 239
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.724359588515654,
+ "seconds": 4.74645025,
+ "prefill_seconds": 0.016314709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.180721375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ },
+ "sent": 244.132262041,
+ "first": 244.314685458,
+ "last": 249.061325375,
+ "chunks": 239
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 262.1868024417305,
+ "ttft_max": 0.019961792000003697,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 262.1737960375258,
+ "seconds": 0.97263725,
+ "prefill_seconds": 0.0194825,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019646792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ },
+ "sent": 249.082937916,
+ "first": 249.102899708,
+ "last": 250.075488708,
+ "chunks": 248
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 622.4916929902471,
+ "ttft_max": 0.07671124999998824,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 157.42673348248232,
+ "seconds": 1.619801125,
+ "prefill_seconds": 0.018772542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03857225,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 250.097818916,
+ "first": 250.136841958,
+ "last": 251.756618958,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 159.27650569847583,
+ "seconds": 1.600989417,
+ "prefill_seconds": 0.01882125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057384666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 250.09795375,
+ "first": 250.155666625,
+ "last": 251.75663625,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 155.6259703736976,
+ "seconds": 1.638544,
+ "prefill_seconds": 0.019652541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01981975,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 250.097881958,
+ "first": 250.118073,
+ "last": 251.756569458,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 161.1673614662196,
+ "seconds": 1.582206209,
+ "prefill_seconds": 0.018784208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076146375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 250.097743833,
+ "first": 250.174455083,
+ "last": 251.756649083,
+ "chunks": 249
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 804.0098677360683,
+ "ttft_max": 0.152253375000015,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.17765117321231,
+ "seconds": 2.495653375,
+ "prefill_seconds": 0.018853667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057652791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 251.780259666,
+ "first": 251.838721958,
+ "last": 254.334342083,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.35011616374786,
+ "seconds": 2.420500416,
+ "prefill_seconds": 0.018860667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.132804792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 251.780316375,
+ "first": 251.913991666,
+ "last": 254.334616541,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.95414641308287,
+ "seconds": 2.476830792,
+ "prefill_seconds": 0.018830958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0764365,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 251.78046625,
+ "first": 251.857560708,
+ "last": 254.334366208,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.00518504450298,
+ "seconds": 2.405542709,
+ "prefill_seconds": 0.018781791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.1515665,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 251.7805225,
+ "first": 251.932775875,
+ "last": 254.3382855,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.65766624393973,
+ "seconds": 2.533339084,
+ "prefill_seconds": 0.01976625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019970708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 251.7803655,
+ "first": 251.801003208,
+ "last": 254.33430275,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.53520523380242,
+ "seconds": 2.439369583,
+ "prefill_seconds": 0.018769417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113946458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 251.780411666,
+ "first": 251.895127958,
+ "last": 254.334824375,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.73698945476035,
+ "seconds": 2.458139583,
+ "prefill_seconds": 0.018799708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.095197542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 251.780645541,
+ "first": 251.87635975,
+ "last": 254.334739291,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.41207868563185,
+ "seconds": 2.514493375,
+ "prefill_seconds": 0.018852125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038842416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 251.780173291,
+ "first": 251.81987025,
+ "last": 254.334317916,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 797.3471530099499,
+ "ttft_max": 0.30202095799998574,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.96634395684408,
+ "seconds": 5.003301791,
+ "prefill_seconds": 0.018808291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131745375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.361831833,
+ "first": 254.4952495,
+ "last": 259.498513041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.5454993116723,
+ "seconds": 4.852937042,
+ "prefill_seconds": 0.018816667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.281690666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.3623815,
+ "first": 254.645720208,
+ "last": 259.499229166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.74514911705729,
+ "seconds": 4.927998167,
+ "prefill_seconds": 0.018785,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.206731125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.361883375,
+ "first": 254.570512166,
+ "last": 259.498974791,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.74997697292843,
+ "seconds": 4.834125333,
+ "prefill_seconds": 0.018753667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.300441,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362455208,
+ "first": 254.664476166,
+ "last": 259.4988315,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.028713371847566,
+ "seconds": 5.097072917,
+ "prefill_seconds": 0.018839959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038308708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.361930791,
+ "first": 254.401221583,
+ "last": 259.498249,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.844636623049375,
+ "seconds": 5.115896459,
+ "prefill_seconds": 0.019437292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019580542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.361976666,
+ "first": 254.382366291,
+ "last": 259.498230083,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.5477064118607,
+ "seconds": 4.946873833,
+ "prefill_seconds": 0.018806416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.188014375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362214791,
+ "first": 254.551724583,
+ "last": 259.498775625,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.587413697805275,
+ "seconds": 5.040779541,
+ "prefill_seconds": 0.018805667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094318667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362335333,
+ "first": 254.457626083,
+ "last": 259.499060041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.141925990882235,
+ "seconds": 4.890498292,
+ "prefill_seconds": 0.01876025,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.244148834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362258791,
+ "first": 254.608091916,
+ "last": 259.499184583,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.2140262100971,
+ "seconds": 5.078262375,
+ "prefill_seconds": 0.018814833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.056949458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362019833,
+ "first": 254.4200285,
+ "last": 259.498265416,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.40008152954365,
+ "seconds": 5.059515625,
+ "prefill_seconds": 0.018778,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.075604375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362145166,
+ "first": 254.438818083,
+ "last": 259.498341875,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.15999607320748,
+ "seconds": 4.984363166,
+ "prefill_seconds": 0.018825,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.150512875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362062166,
+ "first": 254.514063291,
+ "last": 259.499334458,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.775720261940045,
+ "seconds": 5.022085333,
+ "prefill_seconds": 0.018795583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113008042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362104875,
+ "first": 254.476434333,
+ "last": 259.499267458,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.343333200012474,
+ "seconds": 4.871680583,
+ "prefill_seconds": 0.018804125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.262896292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362604041,
+ "first": 254.626896916,
+ "last": 259.498732166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.942677792211114,
+ "seconds": 4.909257875,
+ "prefill_seconds": 0.018807583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.225384334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.362501541,
+ "first": 254.589343833,
+ "last": 259.499018333,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.35307162482252,
+ "seconds": 4.965623125,
+ "prefill_seconds": 0.01884425,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169330792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 254.361755,
+ "first": 254.532916458,
+ "last": 259.498760125,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 347.10809909124464,
+ "ttft_max": 0.01903087499999856,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 347.04676560369313,
+ "seconds": 0.734771291,
+ "prefill_seconds": 0.0181315,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0184205,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 305,
+ "accepted": 166,
+ "acceptance_rate": 0.5442622950819672,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 262.414576583,
+ "first": 262.433607458,
+ "last": 263.168248916,
+ "chunks": 90
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 645.8577525210864,
+ "ttft_max": 0.07029229199997644,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 167.9839637527971,
+ "seconds": 1.518002042,
+ "prefill_seconds": 0.017058292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0694655,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 222,
+ "drafted": 41,
+ "accepted": 33,
+ "acceptance_rate": 0.8048780487804879,
+ "tokens_per_round": 1.1531531531531531
+ },
+ "sent": 263.223389541,
+ "first": 263.293681833,
+ "last": 264.811650125,
+ "chunks": 223
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 165.04391455469687,
+ "seconds": 1.545043334,
+ "prefill_seconds": 0.017076334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.052446625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 263.223573125,
+ "first": 263.276616166,
+ "last": 264.8216225,
+ "chunks": 225
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 161.8603572884961,
+ "seconds": 1.575432084,
+ "prefill_seconds": 0.017927792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.018190708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 263.223487625,
+ "first": 263.242327375,
+ "last": 264.81769975,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 163.64776206179113,
+ "seconds": 1.558224792,
+ "prefill_seconds": 0.017220041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.035387,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 41,
+ "accepted": 32,
+ "acceptance_rate": 0.7804878048780488,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 263.22323575,
+ "first": 263.25954125,
+ "last": 264.817712708,
+ "chunks": 224
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 816.154892487187,
+ "ttft_max": 0.13931495899998936,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.6861953253668,
+ "seconds": 2.412803292,
+ "prefill_seconds": 0.017041708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.087976042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 264.846511833,
+ "first": 264.935187666,
+ "last": 267.348084166,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.22283690948855,
+ "seconds": 2.378224708,
+ "prefill_seconds": 0.016980625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.138756709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 237,
+ "drafted": 27,
+ "accepted": 18,
+ "acceptance_rate": 0.6666666666666666,
+ "tokens_per_round": 1.080168776371308
+ },
+ "sent": 264.846802041,
+ "first": 264.986117,
+ "last": 267.364310708,
+ "chunks": 238
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.21882043800122,
+ "seconds": 2.446774958,
+ "prefill_seconds": 0.01692975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.053928416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 264.846566291,
+ "first": 264.90109175,
+ "last": 267.348025083,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.69546221437821,
+ "seconds": 2.483069792,
+ "prefill_seconds": 0.017370417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017606417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 264.846614541,
+ "first": 264.864785166,
+ "last": 267.347922041,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.50304534879383,
+ "seconds": 2.463695625,
+ "prefill_seconds": 0.019371583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.036972417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 264.846660166,
+ "first": 264.8841655,
+ "last": 267.348166375,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.43608575659778,
+ "seconds": 2.395804,
+ "prefill_seconds": 0.016969625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.104798375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 264.846848041,
+ "first": 264.95216475,
+ "last": 267.348208291,
+ "chunks": 234
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.97931103778156,
+ "seconds": 2.383638458,
+ "prefill_seconds": 0.016975375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.121775625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 264.846755708,
+ "first": 264.969137041,
+ "last": 267.352739,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.94912645421738,
+ "seconds": 2.429748666,
+ "prefill_seconds": 0.017050041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.070941166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ },
+ "sent": 264.846436666,
+ "first": 264.918142291,
+ "last": 267.348244166,
+ "chunks": 234
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 795.3966095266991,
+ "ttft_max": 0.2762710839999727,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.9112888007362,
+ "seconds": 5.109064625,
+ "prefill_seconds": 0.019413458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.036750875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.389845166,
+ "first": 267.427796625,
+ "last": 272.536823041,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.35051860785666,
+ "seconds": 4.871011917,
+ "prefill_seconds": 0.01701525,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.273711042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.389894416,
+ "first": 267.6661655,
+ "last": 272.537800791,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.276274000501004,
+ "seconds": 4.97306025,
+ "prefill_seconds": 0.017041875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.172224,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390209333,
+ "first": 267.564042208,
+ "last": 272.537747166,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.58499563729209,
+ "seconds": 5.0410205,
+ "prefill_seconds": 0.017008042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.10443325,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390263458,
+ "first": 267.4959535,
+ "last": 272.537241583,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.80861373579655,
+ "seconds": 4.921961458,
+ "prefill_seconds": 0.017083208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.223340708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.389942,
+ "first": 267.615203541,
+ "last": 272.53764725,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.101519986395736,
+ "seconds": 4.990066833,
+ "prefill_seconds": 0.017019792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.155221916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390074541,
+ "first": 267.546992875,
+ "last": 272.537043083,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.168243030579276,
+ "seconds": 4.888031208,
+ "prefill_seconds": 0.017024,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2569055,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.389986416,
+ "first": 267.649160958,
+ "last": 272.537388916,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.7224988993842,
+ "seconds": 5.128463083,
+ "prefill_seconds": 0.017152125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017343333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390161708,
+ "first": 267.408377666,
+ "last": 272.53680225,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.629483441703364,
+ "seconds": 4.939038375,
+ "prefill_seconds": 0.017034917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.206259208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390523833,
+ "first": 267.598092291,
+ "last": 272.537894125,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.92779330146235,
+ "seconds": 5.007089125,
+ "prefill_seconds": 0.01701475,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.138409708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390031375,
+ "first": 267.529982708,
+ "last": 272.537136375,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.451851445692355,
+ "seconds": 4.956089875,
+ "prefill_seconds": 0.017006417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.189219167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.3903485,
+ "first": 267.581059916,
+ "last": 272.537550291,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.24548490747553,
+ "seconds": 5.075082875,
+ "prefill_seconds": 0.017020375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.07063525,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390606583,
+ "first": 267.461886208,
+ "last": 272.537164916,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.75611896995513,
+ "seconds": 5.024024791,
+ "prefill_seconds": 0.017000916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.12143275,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.39040525,
+ "first": 267.512956291,
+ "last": 272.537279708,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.07825053627812,
+ "seconds": 5.092030917,
+ "prefill_seconds": 0.017059167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.053735833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390115958,
+ "first": 267.444854166,
+ "last": 272.536891,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.41470917348272,
+ "seconds": 5.058047625,
+ "prefill_seconds": 0.017059,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.087625167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.390450208,
+ "first": 267.47895075,
+ "last": 272.537340916,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.98675938012517,
+ "seconds": 4.905095125,
+ "prefill_seconds": 0.0169305,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.240166167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 267.38977,
+ "first": 267.632118166,
+ "last": 272.537432125,
+ "chunks": 253
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 357.09771979100753,
+ "ttft_max": 0.02024029099999325,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 357.07005135954194,
+ "seconds": 0.714145583,
+ "prefill_seconds": 0.019746083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019916541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 2
+ },
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ },
+ "sent": 272.563330125,
+ "first": 272.583570416,
+ "last": 273.297660666,
+ "chunks": 92
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 618.4479070805222,
+ "ttft_max": 0.07851445800002921,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 156.46971991966768,
+ "seconds": 1.629708292,
+ "prefill_seconds": 0.01944275,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.039390166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 273.322900541,
+ "first": 273.36274,
+ "last": 274.99240275,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 158.3548155302584,
+ "seconds": 1.610307834,
+ "prefill_seconds": 0.019437709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.05881675,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 273.322958916,
+ "first": 273.382180416,
+ "last": 274.992586083,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 160.284594235288,
+ "seconds": 1.590920208,
+ "prefill_seconds": 0.019369333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.078157791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 273.323042125,
+ "first": 273.401556583,
+ "last": 274.992435333,
+ "chunks": 250
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 154.6267272904501,
+ "seconds": 1.64913275,
+ "prefill_seconds": 0.019795417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019949166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ },
+ "sent": 273.3228215,
+ "first": 273.343296,
+ "last": 274.992382833,
+ "chunks": 250
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 794.1760553413691,
+ "ttft_max": 0.15879825000001802,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.43158275989286,
+ "seconds": 2.465397833,
+ "prefill_seconds": 0.019398875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119088542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 275.018285458,
+ "first": 275.138415416,
+ "last": 277.604131583,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.2490729611327,
+ "seconds": 2.446064917,
+ "prefill_seconds": 0.01933225,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.138377875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 275.018488125,
+ "first": 275.157752625,
+ "last": 277.603875083,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.83034555279177,
+ "seconds": 2.504165125,
+ "prefill_seconds": 0.019374167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.080340625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 275.018336666,
+ "first": 275.099585541,
+ "last": 277.603806583,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.91587267057669,
+ "seconds": 2.430518791,
+ "prefill_seconds": 0.019425,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.157737583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 255,
+ "drafted": 6,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ },
+ "sent": 275.018384416,
+ "first": 275.177182666,
+ "last": 277.607662291,
+ "chunks": 256
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.0498251393604,
+ "seconds": 2.523507583,
+ "prefill_seconds": 0.019428208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.061057208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 275.01854475,
+ "first": 275.080205,
+ "last": 277.603754291,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.62420660546744,
+ "seconds": 2.484793875,
+ "prefill_seconds": 0.019429167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.099766709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 275.018601625,
+ "first": 275.119014791,
+ "last": 277.603847833,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.27760510072491,
+ "seconds": 2.542940667,
+ "prefill_seconds": 0.021809916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.041732958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 275.018434791,
+ "first": 275.060769916,
+ "last": 277.603710041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 99.42550415277093,
+ "seconds": 2.564734292,
+ "prefill_seconds": 0.019841125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.020004125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 275.0182115,
+ "first": 275.038962333,
+ "last": 277.603698083,
+ "chunks": 255
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 788.8548028896271,
+ "ttft_max": 0.31118087500004776,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.06330676926831,
+ "seconds": 5.093550875,
+ "prefill_seconds": 0.019405334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.096908417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635270458,
+ "first": 277.733827041,
+ "last": 282.82798875,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.83831146504552,
+ "seconds": 4.919141708,
+ "prefill_seconds": 0.019410667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.270784958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635318875,
+ "first": 277.908384208,
+ "last": 282.82771525,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.249450247420356,
+ "seconds": 4.880434125,
+ "prefill_seconds": 0.019322542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.309351459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635929833,
+ "first": 277.947110708,
+ "last": 282.828041416,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.25550159125926,
+ "seconds": 5.074071334,
+ "prefill_seconds": 0.019411417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.116295708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635370625,
+ "first": 277.753237958,
+ "last": 282.827383791,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.43216216799861,
+ "seconds": 4.957987167,
+ "prefill_seconds": 0.01937825,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.232349833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635884291,
+ "first": 277.869507416,
+ "last": 282.827847833,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.04327510575244,
+ "seconds": 4.8997685,
+ "prefill_seconds": 0.019391209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.290074292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635418666,
+ "first": 277.927773666,
+ "last": 282.827945541,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.83702243367445,
+ "seconds": 5.016029417,
+ "prefill_seconds": 0.019409708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.174288833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.63567375,
+ "first": 277.811379125,
+ "last": 282.827585208,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.6411701691176,
+ "seconds": 5.035428667,
+ "prefill_seconds": 0.019312167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.154905541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635601791,
+ "first": 277.791968875,
+ "last": 282.828079333,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.63429242934714,
+ "seconds": 4.938578375,
+ "prefill_seconds": 0.019462375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.251585334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635465666,
+ "first": 277.888973083,
+ "last": 282.827899125,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.23187804161023,
+ "seconds": 4.97736975,
+ "prefill_seconds": 0.019391084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.212965209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.636000208,
+ "first": 277.850126291,
+ "last": 282.828285458,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.68577118614894,
+ "seconds": 5.132254042,
+ "prefill_seconds": 0.019380583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.058375583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635724208,
+ "first": 277.695039291,
+ "last": 282.827327833,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.499774448454225,
+ "seconds": 5.151538625,
+ "prefill_seconds": 0.019430542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.039145041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635511791,
+ "first": 277.675665916,
+ "last": 282.827238041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.0330804899227,
+ "seconds": 4.996758917,
+ "prefill_seconds": 0.019350167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.193647417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.63577375,
+ "first": 277.830735541,
+ "last": 282.827683125,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.447223289623366,
+ "seconds": 5.054787625,
+ "prefill_seconds": 0.019404584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.135652625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635556958,
+ "first": 277.772669291,
+ "last": 282.8278115,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.31379809341647,
+ "seconds": 5.170966542,
+ "prefill_seconds": 0.01954675,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01973825,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.6358325,
+ "first": 277.656231,
+ "last": 282.827159041,
+ "chunks": 255
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 49.87444922024807,
+ "seconds": 5.112838417,
+ "prefill_seconds": 0.019368834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.077582083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ },
+ "sent": 277.635199416,
+ "first": 277.7144185,
+ "last": 282.827228416,
+ "chunks": 255
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 290.6181341038439,
+ "ttft_max": 0.016731250000020736,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 290.60274750059926,
+ "seconds": 0.877486542,
+ "prefill_seconds": 0.01636925,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.016472375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 284.939780666,
+ "first": 284.956511916,
+ "last": 285.833952,
+ "chunks": 217
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 651.7155973832075,
+ "ttft_max": 0.0660668329999794,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 163.32064822925992,
+ "seconds": 1.56134575,
+ "prefill_seconds": 0.016514084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.016673583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 285.862577625,
+ "first": 285.879703041,
+ "last": 287.441005416,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 168.18952852945154,
+ "seconds": 1.516146708,
+ "prefill_seconds": 0.016321125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.065632417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 285.862633,
+ "first": 285.928699833,
+ "last": 287.444802916,
+ "chunks": 225
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 166.80816283555382,
+ "seconds": 1.528702167,
+ "prefill_seconds": 0.016336208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.04929975,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 285.86270525,
+ "first": 285.912371416,
+ "last": 287.441028291,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 165.0461489711835,
+ "seconds": 1.545022417,
+ "prefill_seconds": 0.016329833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.032998208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 285.862509541,
+ "first": 285.896034666,
+ "last": 287.441016958,
+ "chunks": 224
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 831.0161994582763,
+ "ttft_max": 0.13210483299997122,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.05744545337508,
+ "seconds": 2.381898792,
+ "prefill_seconds": 0.016335083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082185334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 287.473703583,
+ "first": 287.556703833,
+ "last": 289.938711166,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.20361821466938,
+ "seconds": 2.447131917,
+ "prefill_seconds": 0.016739709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01690125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 287.473754208,
+ "first": 287.491339583,
+ "last": 289.938426125,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.5428566352171,
+ "seconds": 2.349302459,
+ "prefill_seconds": 0.016337209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.114814833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 287.473905833,
+ "first": 287.5893835,
+ "last": 289.938776916,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.33209910232699,
+ "seconds": 2.398146958,
+ "prefill_seconds": 0.0163225,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.065811875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 287.473959458,
+ "first": 287.540367791,
+ "last": 289.93848275,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.90300174797198,
+ "seconds": 2.430817,
+ "prefill_seconds": 0.016336208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033205166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 287.473802458,
+ "first": 287.507666291,
+ "last": 289.938498916,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.79540623645326,
+ "seconds": 2.365592458,
+ "prefill_seconds": 0.016332541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098416125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 287.474046916,
+ "first": 287.57303225,
+ "last": 289.938839,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.61353180011204,
+ "seconds": 2.414463333,
+ "prefill_seconds": 0.016364,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049551,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 287.47384725,
+ "first": 287.524044458,
+ "last": 289.938604958,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.95265378984647,
+ "seconds": 2.340466167,
+ "prefill_seconds": 0.0163505,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131109708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ },
+ "sent": 287.473629625,
+ "first": 287.605734458,
+ "last": 289.946165375,
+ "chunks": 237
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 830.5125280653175,
+ "ttft_max": 0.26301050000000714,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.97328212515975,
+ "seconds": 4.813747417,
+ "prefill_seconds": 0.016346375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.114878917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.974824875,
+ "first": 290.091412583,
+ "last": 294.905738875,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.15373023346543,
+ "seconds": 4.797405542,
+ "prefill_seconds": 0.016338667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131151584,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.974872666,
+ "first": 290.10773875,
+ "last": 294.905436208,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.88709164860404,
+ "seconds": 4.732116583,
+ "prefill_seconds": 0.016302834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.196179958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.9752195,
+ "first": 290.17302875,
+ "last": 294.905335416,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.335340783530825,
+ "seconds": 4.781070042,
+ "prefill_seconds": 0.016332584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.147376375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975468583,
+ "first": 290.124080791,
+ "last": 294.905142458,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.26651254634462,
+ "seconds": 4.878840917,
+ "prefill_seconds": 0.016349958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049839875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.974923291,
+ "first": 290.026090333,
+ "last": 294.904896666,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.26131145645951,
+ "seconds": 4.699480959,
+ "prefill_seconds": 0.01632775,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.228727,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.974972458,
+ "first": 290.205723375,
+ "last": 294.905556041,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.44905981674593,
+ "seconds": 4.683276458,
+ "prefill_seconds": 0.016322208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.245048459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975297416,
+ "first": 290.222047041,
+ "last": 294.905838458,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.0919816449772,
+ "seconds": 4.895187166,
+ "prefill_seconds": 0.016356292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033592792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975163875,
+ "first": 290.009740291,
+ "last": 294.904886375,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.91876704520609,
+ "seconds": 4.911518792,
+ "prefill_seconds": 0.017077209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017252375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975518458,
+ "first": 289.993361416,
+ "last": 294.90485675,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.79500372929414,
+ "seconds": 4.8300025,
+ "prefill_seconds": 0.016333958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098574666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975023208,
+ "first": 290.075069666,
+ "last": 294.905689416,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.6172965279471,
+ "seconds": 4.846315125,
+ "prefill_seconds": 0.0162985,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082317875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975656375,
+ "first": 290.058719583,
+ "last": 294.9059905,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.441649342831724,
+ "seconds": 4.862547292,
+ "prefill_seconds": 0.016325667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066120916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975071166,
+ "first": 290.042422,
+ "last": 294.9055095,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.64040514687041,
+ "seconds": 4.666876084,
+ "prefill_seconds": 0.016307709,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.261180375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975348291,
+ "first": 290.238358791,
+ "last": 294.905790916,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.518162910808236,
+ "seconds": 4.764737542,
+ "prefill_seconds": 0.016333209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.163652334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.97511825,
+ "first": 290.140429708,
+ "last": 294.905645291,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.70165697366044,
+ "seconds": 4.748456833,
+ "prefill_seconds": 0.016300625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.17993075,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.975405166,
+ "first": 290.15672825,
+ "last": 294.905898458,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.0732705539128,
+ "seconds": 4.7158235,
+ "prefill_seconds": 0.0163535,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.212464833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 289.974755416,
+ "first": 290.189391083,
+ "last": 294.905949541,
+ "chunks": 240
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 262.3964081747341,
+ "ttft_max": 0.0202437500000201,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 262.3845845455222,
+ "seconds": 0.971855875,
+ "prefill_seconds": 0.019778708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01992125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ },
+ "sent": 294.935785083,
+ "first": 294.956028833,
+ "last": 295.927840916,
+ "chunks": 248
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 622.917485317143,
+ "ttft_max": 0.07658683299996483,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 161.28302862963503,
+ "seconds": 1.5810715,
+ "prefill_seconds": 0.018786167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076180834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 295.956686583,
+ "first": 296.033273416,
+ "last": 297.614340125,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 159.38804162765362,
+ "seconds": 1.599869083,
+ "prefill_seconds": 0.018792791,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057388042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 295.956737458,
+ "first": 296.01448975,
+ "last": 297.614322583,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 157.53843007753804,
+ "seconds": 1.618652667,
+ "prefill_seconds": 0.018812834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03856325,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 295.956797541,
+ "first": 295.995684208,
+ "last": 297.614334,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 155.72805176609543,
+ "seconds": 1.637469917,
+ "prefill_seconds": 0.019629625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.01979225,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 295.9566095,
+ "first": 295.976884083,
+ "last": 297.614306875,
+ "chunks": 249
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 804.202708162487,
+ "ttft_max": 0.15177995799996324,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.68020664965388,
+ "seconds": 2.532771917,
+ "prefill_seconds": 0.019620959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019774875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 297.64415175,
+ "first": 297.664605583,
+ "last": 300.197336166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.43305010971675,
+ "seconds": 2.5139735,
+ "prefill_seconds": 0.018797042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038582458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 297.644201708,
+ "first": 297.683404083,
+ "last": 300.197378416,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.55656664589235,
+ "seconds": 2.438871208,
+ "prefill_seconds": 0.018806958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113656292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 297.644494291,
+ "first": 297.758623166,
+ "last": 300.197750125,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.36886563708484,
+ "seconds": 2.420069709,
+ "prefill_seconds": 0.01880625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.132486167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 297.64434525,
+ "first": 297.777433,
+ "last": 300.197624833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.02447525034125,
+ "seconds": 2.405105042,
+ "prefill_seconds": 0.018772417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.151229541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 297.644441958,
+ "first": 297.796221916,
+ "last": 300.201279458,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.19631740884425,
+ "seconds": 2.495197542,
+ "prefill_seconds": 0.018787666,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0573445,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 297.644294041,
+ "first": 297.702194083,
+ "last": 300.197352166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.97124018309296,
+ "seconds": 2.476419625,
+ "prefill_seconds": 0.018807417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076114708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 297.644248791,
+ "first": 297.720997833,
+ "last": 300.197710375,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.75857642910877,
+ "seconds": 2.457628167,
+ "prefill_seconds": 0.018811125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094907375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 297.644084541,
+ "first": 297.739818708,
+ "last": 300.197828625,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 800.4553963396994,
+ "ttft_max": 0.30195220800004563,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.759322934436305,
+ "seconds": 4.833269,
+ "prefill_seconds": 0.018797542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.281478292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.231652083,
+ "first": 300.51546325,
+ "last": 305.348800291,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.22411279350947,
+ "seconds": 5.0772425,
+ "prefill_seconds": 0.018789584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03817825,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.231700833,
+ "first": 300.271134916,
+ "last": 305.348341833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.039204735749585,
+ "seconds": 5.09600425,
+ "prefill_seconds": 0.019426541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019594541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.232012,
+ "first": 300.252341125,
+ "last": 305.348313,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.95107702209289,
+ "seconds": 4.908464167,
+ "prefill_seconds": 0.018742917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.206534083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.232607625,
+ "first": 300.440241875,
+ "last": 305.349147916,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.16930216100543,
+ "seconds": 4.983456667,
+ "prefill_seconds": 0.0187925,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.1318505,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.231747833,
+ "first": 300.365159166,
+ "last": 305.349047041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.15083463192746,
+ "seconds": 4.889662875,
+ "prefill_seconds": 0.018801,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.225291708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.23179275,
+ "first": 300.459042875,
+ "last": 305.348887208,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.353013017002034,
+ "seconds": 4.870779833,
+ "prefill_seconds": 0.018853834,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.244087125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.232240375,
+ "first": 300.477911166,
+ "last": 305.349439625,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.55742295669859,
+ "seconds": 4.945941542,
+ "prefill_seconds": 0.018782875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.232067208,
+ "first": 300.402732291,
+ "last": 305.349306333,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.36257397740955,
+ "seconds": 4.964704458,
+ "prefill_seconds": 0.018785167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.150635792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.232163083,
+ "first": 300.38394475,
+ "last": 305.349396666,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.787357994357095,
+ "seconds": 5.020934541,
+ "prefill_seconds": 0.01880575,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094437167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.23196425,
+ "first": 300.327546375,
+ "last": 305.348466166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.41054903674682,
+ "seconds": 5.058465041,
+ "prefill_seconds": 0.018778792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.05694,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.231836916,
+ "first": 300.289925708,
+ "last": 305.348357458,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.97766116066705,
+ "seconds": 5.002191042,
+ "prefill_seconds": 0.018801375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113120959,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.231921875,
+ "first": 300.34635125,
+ "last": 305.348693541,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.59845446028725,
+ "seconds": 5.039679625,
+ "prefill_seconds": 0.018817625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.075707583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.232111583,
+ "first": 300.308746625,
+ "last": 305.348481875,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.55521388941559,
+ "seconds": 4.85204,
+ "prefill_seconds": 0.018757833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.262793375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.231878,
+ "first": 300.49667725,
+ "last": 305.349356333,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.96454363918808,
+ "seconds": 4.814541625,
+ "prefill_seconds": 0.018778625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.3002615,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.232298208,
+ "first": 300.534250416,
+ "last": 305.348984708,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.754304328772875,
+ "seconds": 4.927126416,
+ "prefill_seconds": 0.018756542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.187937709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 300.231581625,
+ "first": 300.421495166,
+ "last": 305.3491995,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 290.11219414456707,
+ "ttft_max": 0.016669167000031848,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 290.0973561038901,
+ "seconds": 0.87901525,
+ "prefill_seconds": 0.016310084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.016434416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ },
+ "sent": 308.221080041,
+ "first": 308.237749208,
+ "last": 309.1167195,
+ "chunks": 217
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 651.566298308141,
+ "ttft_max": 0.0668896669999981,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 165.01353390530363,
+ "seconds": 1.545327792,
+ "prefill_seconds": 0.016315083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033670666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 309.150666166,
+ "first": 309.184809333,
+ "last": 310.730107083,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 163.28918201965183,
+ "seconds": 1.561646625,
+ "prefill_seconds": 0.017215208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017364709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 309.150722833,
+ "first": 309.168494708,
+ "last": 310.730092875,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 166.7782701067951,
+ "seconds": 1.528976166,
+ "prefill_seconds": 0.01635025,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049985708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ },
+ "sent": 309.150815208,
+ "first": 309.201149708,
+ "last": 310.730115875,
+ "chunks": 224
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 168.14815632964886,
+ "seconds": 1.51651975,
+ "prefill_seconds": 0.0163105,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066292083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ },
+ "sent": 309.150580458,
+ "first": 309.217470125,
+ "last": 310.733953208,
+ "chunks": 225
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 830.9803317342685,
+ "ttft_max": 0.1320373749999817,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.89801571795324,
+ "seconds": 2.430932542,
+ "prefill_seconds": 0.016297417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033722542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 310.766304708,
+ "first": 310.8007125,
+ "last": 313.231627958,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.78214943631772,
+ "seconds": 2.365883417,
+ "prefill_seconds": 0.016259416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098723833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 310.76635525,
+ "first": 310.865833791,
+ "last": 313.232120041,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.31822267581995,
+ "seconds": 2.398459959,
+ "prefill_seconds": 0.016258,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.066167792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 310.766503166,
+ "first": 310.833245791,
+ "last": 313.232074625,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 107.043218122123,
+ "seconds": 2.382215375,
+ "prefill_seconds": 0.016325084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082435417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 310.76645175,
+ "first": 310.849569666,
+ "last": 313.231988041,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.93092430449789,
+ "seconds": 2.340933042,
+ "prefill_seconds": 0.016338959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.131257625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ },
+ "sent": 310.766405625,
+ "first": 310.898443,
+ "last": 313.239348458,
+ "chunks": 237
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 108.52330130464998,
+ "seconds": 2.349725791,
+ "prefill_seconds": 0.016277667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.114925291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 310.766615375,
+ "first": 310.882109,
+ "last": 313.23203375,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.19956955362132,
+ "seconds": 2.447227,
+ "prefill_seconds": 0.017238375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017401583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 310.76654975,
+ "first": 310.784416708,
+ "last": 313.231598375,
+ "chunks": 235
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.6030839121993,
+ "seconds": 2.414702209,
+ "prefill_seconds": 0.016262292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049953875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ },
+ "sent": 310.766232208,
+ "first": 310.816983458,
+ "last": 313.231833333,
+ "chunks": 235
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 831.0121583537492,
+ "ttft_max": 0.2618807499999889,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.95027045081604,
+ "seconds": 4.908540375,
+ "prefill_seconds": 0.016921542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.017110583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.272857333,
+ "first": 313.291140416,
+ "last": 318.199646541,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.12312565623399,
+ "seconds": 4.89226225,
+ "prefill_seconds": 0.016278375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.033401167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.272908541,
+ "first": 313.307429333,
+ "last": 318.199663041,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.66534460310047,
+ "seconds": 4.664746959,
+ "prefill_seconds": 0.016233084,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.260031625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.273462375,
+ "first": 313.535343125,
+ "last": 318.200459708,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.730871070835924,
+ "seconds": 4.745875042,
+ "prefill_seconds": 0.016297833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.179193292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.272953875,
+ "first": 313.45405375,
+ "last": 318.200762041,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.47625756790819,
+ "seconds": 4.680938291,
+ "prefill_seconds": 0.016249583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.243827667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.273226625,
+ "first": 313.519107333,
+ "last": 318.200277708,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.184599992366444,
+ "seconds": 4.794621,
+ "prefill_seconds": 0.01626375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.130646833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.273403583,
+ "first": 313.405226166,
+ "last": 318.200390541,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.36413897051653,
+ "seconds": 4.778489917,
+ "prefill_seconds": 0.016247625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.146778292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.27350225,
+ "first": 313.421475666,
+ "last": 318.200350208,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.47296465536849,
+ "seconds": 4.859645375,
+ "prefill_seconds": 0.016370417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.065873542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.273087,
+ "first": 313.340081291,
+ "last": 318.199738166,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.00491396626079,
+ "seconds": 4.810874708,
+ "prefill_seconds": 0.016333583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.114492333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.272999583,
+ "first": 313.388962916,
+ "last": 318.200301416,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.64890537069318,
+ "seconds": 4.843405541,
+ "prefill_seconds": 0.016271167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.082010875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.273128125,
+ "first": 313.35635825,
+ "last": 318.200141375,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.915120953167545,
+ "seconds": 4.729656458,
+ "prefill_seconds": 0.016279875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.19541025,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.27316975,
+ "first": 313.470335083,
+ "last": 318.200551041,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.82363565611979,
+ "seconds": 4.8273845,
+ "prefill_seconds": 0.016271416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.098151458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.273042916,
+ "first": 313.372629041,
+ "last": 318.200660833,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.10040781099566,
+ "seconds": 4.713458,
+ "prefill_seconds": 0.016259792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.211572583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.273303208,
+ "first": 313.486597958,
+ "last": 318.200508166,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.54590708472722,
+ "seconds": 4.76226875,
+ "prefill_seconds": 0.016263292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.163035166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.273360625,
+ "first": 313.4377435,
+ "last": 318.20042525,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 54.28749844480688,
+ "seconds": 4.697214042,
+ "prefill_seconds": 0.016251375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.2276415,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.2736095,
+ "first": 313.502852125,
+ "last": 318.200815875,
+ "chunks": 240
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.296831998913866,
+ "seconds": 4.876012375,
+ "prefill_seconds": 0.016277333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.049526583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "dcc30ceea29d",
+ "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 239,
+ "drafted": 18,
+ "accepted": 16,
+ "acceptance_rate": 0.8888888888888888,
+ "tokens_per_round": 1.0711297071129706
+ },
+ "sent": 313.272782291,
+ "first": 313.323708,
+ "last": 318.19969325,
+ "chunks": 240
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 262.21991974165917,
+ "ttft_max": 0.019898124999997435,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 262.2083813625824,
+ "seconds": 0.972508959,
+ "prefill_seconds": 0.019450166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019591042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ },
+ "sent": 318.232904875,
+ "first": 318.252803,
+ "last": 319.225269166,
+ "chunks": 248
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 622.948506893192,
+ "ttft_max": 0.07660004199999548,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 157.54933154086103,
+ "seconds": 1.618540666,
+ "prefill_seconds": 0.018821792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038648084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 319.259709291,
+ "first": 319.298847208,
+ "last": 320.9173675,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 161.28251001913515,
+ "seconds": 1.581076584,
+ "prefill_seconds": 0.018785584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076145417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 319.259760166,
+ "first": 319.336360208,
+ "last": 320.917396375,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 155.73814530505885,
+ "seconds": 1.637363791,
+ "prefill_seconds": 0.019692083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019849542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 319.259818458,
+ "first": 319.280021875,
+ "last": 320.917341833,
+ "chunks": 249
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 159.39187730854874,
+ "seconds": 1.599830583,
+ "prefill_seconds": 0.018723917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057366708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ },
+ "sent": 319.259639291,
+ "first": 319.317571416,
+ "last": 320.917359416,
+ "chunks": 249
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 8,
+ "aggregate_decode_tps": 804.3912665584089,
+ "ttft_max": 0.1521238329999619,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.9992905473494,
+ "seconds": 2.475745208,
+ "prefill_seconds": 0.018804291,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076227416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 320.951588666,
+ "first": 321.028649208,
+ "last": 323.504430833,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 106.0499026134387,
+ "seconds": 2.404528375,
+ "prefill_seconds": 0.018741125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.150904375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ },
+ "sent": 320.951646583,
+ "first": 321.103770416,
+ "last": 323.508267916,
+ "chunks": 253
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 103.78672128658181,
+ "seconds": 2.456961708,
+ "prefill_seconds": 0.018798958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094958666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 320.951808583,
+ "first": 321.04744825,
+ "last": 323.504369916,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 102.22434666240264,
+ "seconds": 2.494513375,
+ "prefill_seconds": 0.01880375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057425166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 320.951908666,
+ "first": 321.009842,
+ "last": 323.504324,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 104.58395403722076,
+ "seconds": 2.438232541,
+ "prefill_seconds": 0.018791667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.1136865,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 320.951854583,
+ "first": 321.066244125,
+ "last": 323.504529625,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 101.45915505061564,
+ "seconds": 2.513326667,
+ "prefill_seconds": 0.018847541,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03865225,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 320.951702833,
+ "first": 320.991028,
+ "last": 323.504351708,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 100.7042236486789,
+ "seconds": 2.532167875,
+ "prefill_seconds": 0.019715959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019900375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 320.9517515,
+ "first": 320.972188666,
+ "last": 323.504311291,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 105.39538069612306,
+ "seconds": 2.419460875,
+ "prefill_seconds": 0.018780292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.132466625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 320.951514833,
+ "first": 321.085027083,
+ "last": 323.50478975,
+ "chunks": 252
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "chat",
+ "streams": 16,
+ "aggregate_decode_tps": 801.2450260525325,
+ "ttft_max": 0.3025994579999747,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.08908039467293,
+ "seconds": 5.090929959,
+ "prefill_seconds": 0.019595375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.019816333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542316958,
+ "first": 323.563395791,
+ "last": 328.654280083,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.27460697129549,
+ "seconds": 5.072143083,
+ "prefill_seconds": 0.0188055,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.038602,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542368708,
+ "first": 323.582208666,
+ "last": 328.654311625,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.648919005256076,
+ "seconds": 5.034658291,
+ "prefill_seconds": 0.018770458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.075997792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.54271425,
+ "first": 323.619719958,
+ "last": 328.654323666,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.809909127089995,
+ "seconds": 4.82863925,
+ "prefill_seconds": 0.018726458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.281503583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542420375,
+ "first": 323.826029125,
+ "last": 328.655109291,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.40358953285565,
+ "seconds": 4.866078875,
+ "prefill_seconds": 0.018754083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.244145875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542962541,
+ "first": 323.788552166,
+ "last": 328.654811041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.837700640704554,
+ "seconds": 5.0159625,
+ "prefill_seconds": 0.018772625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.094714792,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542471,
+ "first": 323.638498125,
+ "last": 328.654887041,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.605931519919196,
+ "seconds": 4.847362125,
+ "prefill_seconds": 0.018756417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.262877833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542769458,
+ "first": 323.80731225,
+ "last": 328.654937416,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.20315081329055,
+ "seconds": 4.884762625,
+ "prefill_seconds": 0.018713334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.225421875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542519916,
+ "first": 323.769792833,
+ "last": 328.654603416,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 50.46094673811927,
+ "seconds": 5.053412916,
+ "prefill_seconds": 0.018739,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.057202375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.543029958,
+ "first": 323.600936041,
+ "last": 328.655159375,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 52.00380915298769,
+ "seconds": 4.903486959,
+ "prefill_seconds": 0.018685375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.206730875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542568375,
+ "first": 323.751071833,
+ "last": 328.654986166,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.41379407069505,
+ "seconds": 4.959758458,
+ "prefill_seconds": 0.018779375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.15092825,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542815041,
+ "first": 323.69485825,
+ "last": 328.655296208,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.60879976933287,
+ "seconds": 4.941017833,
+ "prefill_seconds": 0.018750542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.169453166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542615541,
+ "first": 323.713606541,
+ "last": 328.655200291,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.805692932797264,
+ "seconds": 4.922238958,
+ "prefill_seconds": 0.01877275,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.188166209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542900833,
+ "first": 323.732385958,
+ "last": 328.654701541,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.2207366147053,
+ "seconds": 4.978452417,
+ "prefill_seconds": 0.018818875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.13210325,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542660416,
+ "first": 323.676052791,
+ "last": 328.655355208,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 51.02788546370878,
+ "seconds": 4.997267625,
+ "prefill_seconds": 0.018750125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113471917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542856125,
+ "first": 323.657248791,
+ "last": 328.654717333,
+ "chunks": 252
+ },
+ {
+ "usage": {
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "total_tokens": 274,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 53.01680247008635,
+ "seconds": 4.809795916,
+ "prefill_seconds": 0.018807708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.300145291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "756f2e358866",
+ "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 251,
+ "drafted": 8,
+ "accepted": 4,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ },
+ "sent": 323.542239333,
+ "first": 323.844838791,
+ "last": 328.655471083,
+ "chunks": 252
+ }
+ ]
+ }
+ ],
+ "performance-qwen": [
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 211.92194615871247,
+ "ttft_max": 0.03147329199998694,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 211.73019910553273,
+ "seconds": 1.204362916,
+ "prefill_seconds": 0.030946833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.031178042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 43,
+ "drafted": 658,
+ "accepted": 213,
+ "acceptance_rate": 0.32370820668693007,
+ "tokens_per_round": 5.953488372093023
+ },
+ "sent": 220.468364583,
+ "first": 220.499837875,
+ "last": 221.703111083,
+ "chunks": 44
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 379.28424450667757,
+ "ttft_max": 0.15881983299999547,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 97.82656232856375,
+ "seconds": 2.606654,
+ "prefill_seconds": 0.117248833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.117549,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 221.705125833,
+ "first": 221.823199625,
+ "last": 224.429784583,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 99.3859749337853,
+ "seconds": 2.565754375,
+ "prefill_seconds": 0.123503708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.158455209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 221.705284458,
+ "first": 221.864104291,
+ "last": 224.429873916,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 96.30150148406732,
+ "seconds": 2.647933792,
+ "prefill_seconds": 0.075959041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076291,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 221.705184541,
+ "first": 221.78189875,
+ "last": 224.429773291,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 94.82365861929128,
+ "seconds": 2.689202291,
+ "prefill_seconds": 0.034662459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.034964041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 221.705046708,
+ "first": 221.740597958,
+ "last": 224.429743875,
+ "chunks": 68
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 140.9551661001172,
+ "ttft_max": 0.03612595900000315,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 140.8800278113749,
+ "seconds": 1.81005075,
+ "prefill_seconds": 0.035580708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03579525,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 975,
+ "accepted": 191,
+ "acceptance_rate": 0.19589743589743588,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 224.430958291,
+ "first": 224.46708425,
+ "last": 226.276170125,
+ "chunks": 66
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 314.3405067152877,
+ "ttft_max": 0.15906683299999713,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 80.5539987986341,
+ "seconds": 3.165578417,
+ "prefill_seconds": 0.079156209,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119072042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 226.278272291,
+ "first": 226.397800083,
+ "last": 229.563340541,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 81.57684958648021,
+ "seconds": 3.125886833,
+ "prefill_seconds": 0.118852042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.158741084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 226.278422708,
+ "first": 226.437489541,
+ "last": 229.563531708,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 78.59021380751824,
+ "seconds": 3.244678792,
+ "prefill_seconds": 0.039655625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03991425,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 226.278334458,
+ "first": 226.318642875,
+ "last": 229.563267041,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 79.5624005567496,
+ "seconds": 3.2050315,
+ "prefill_seconds": 0.079313625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.079552834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 226.278194875,
+ "first": 226.358296083,
+ "last": 229.563297333,
+ "chunks": 90
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 212.45570139283868,
+ "ttft_max": 0.03133441599999287,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 212.2655129411728,
+ "seconds": 1.201325625,
+ "prefill_seconds": 0.030853583,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03105075,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 43,
+ "drafted": 658,
+ "accepted": 213,
+ "acceptance_rate": 0.32370820668693007,
+ "tokens_per_round": 5.953488372093023
+ },
+ "sent": 240.36544375,
+ "first": 240.396778166,
+ "last": 241.597028375,
+ "chunks": 44
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 377.1968654072174,
+ "ttft_max": 0.15561083300002565,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 97.27497352907643,
+ "seconds": 2.621434792,
+ "prefill_seconds": 0.082676,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.113711334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 255,
+ "accepted": 188,
+ "acceptance_rate": 0.7372549019607844,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 241.599939541,
+ "first": 241.714227666,
+ "last": 244.335596875,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 94.30269209545193,
+ "seconds": 2.704058541,
+ "prefill_seconds": 0.030840083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.031085166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 251,
+ "accepted": 188,
+ "acceptance_rate": 0.749003984063745,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 241.60006575,
+ "first": 241.631526791,
+ "last": 244.335539708,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 98.83527473590149,
+ "seconds": 2.5800505,
+ "prefill_seconds": 0.12406575,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.155116875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 255,
+ "accepted": 188,
+ "acceptance_rate": 0.7372549019607844,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 241.599997125,
+ "first": 241.755607958,
+ "last": 244.335685,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 95.77020109254082,
+ "seconds": 2.662623625,
+ "prefill_seconds": 0.041446,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.07251225,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 255,
+ "accepted": 188,
+ "acceptance_rate": 0.7372549019607844,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 241.599864458,
+ "first": 241.672987208,
+ "last": 244.335552958,
+ "chunks": 68
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 141.19484385798825,
+ "ttft_max": 0.036089042000014615,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 141.11870149437556,
+ "seconds": 1.806989416,
+ "prefill_seconds": 0.035520292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0357655,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 975,
+ "accepted": 191,
+ "acceptance_rate": 0.19589743589743588,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 244.337321541,
+ "first": 244.373410583,
+ "last": 246.179425541,
+ "chunks": 66
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 316.415283652713,
+ "ttft_max": 0.15867833299998324,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 79.10471251890549,
+ "seconds": 3.223575333,
+ "prefill_seconds": 0.039642875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.039891958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 246.182093458,
+ "first": 246.222425833,
+ "last": 249.445940416,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 82.12139861437048,
+ "seconds": 3.105159,
+ "prefill_seconds": 0.118494625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.15834425,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 246.182233208,
+ "first": 246.340911541,
+ "last": 249.4460375,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 81.08921362016,
+ "seconds": 3.144684584,
+ "prefill_seconds": 0.118604958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.118837791,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 246.182148166,
+ "first": 246.301384375,
+ "last": 249.446024291,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 80.08463427060869,
+ "seconds": 3.184131417,
+ "prefill_seconds": 0.079125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.07937475,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 246.18201175,
+ "first": 246.261895833,
+ "last": 249.445976166,
+ "chunks": 89
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 211.59088653158616,
+ "ttft_max": 0.0313943330000086,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 211.40970570965607,
+ "seconds": 1.206188709,
+ "prefill_seconds": 0.030824792,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.031058833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 43,
+ "drafted": 658,
+ "accepted": 213,
+ "acceptance_rate": 0.32370820668693007,
+ "tokens_per_round": 5.953488372093023
+ },
+ "sent": 254.012542333,
+ "first": 254.043936666,
+ "last": 255.249092541,
+ "chunks": 44
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 377.49507004207,
+ "ttft_max": 0.16005191600001467,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 95.84680597070513,
+ "seconds": 2.660495542,
+ "prefill_seconds": 0.076353833,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.076651,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 255.252503833,
+ "first": 255.329673833,
+ "last": 257.9901095,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 97.3572455490473,
+ "seconds": 2.619219541,
+ "prefill_seconds": 0.11765925,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.117936625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 255.252631541,
+ "first": 255.370974916,
+ "last": 257.990136916,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 94.37434527797964,
+ "seconds": 2.7020055,
+ "prefill_seconds": 0.034829917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.035106667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 255.252561583,
+ "first": 255.288130666,
+ "last": 257.990094125,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 98.92393069777353,
+ "seconds": 2.57773825,
+ "prefill_seconds": 0.12433075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.159474,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 255.252426125,
+ "first": 255.412478041,
+ "last": 257.990152708,
+ "chunks": 68
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 140.8142774005683,
+ "ttft_max": 0.03615183299996261,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 140.73994300998172,
+ "seconds": 1.811852375,
+ "prefill_seconds": 0.035574334,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.035819583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 975,
+ "accepted": 191,
+ "acceptance_rate": 0.19589743589743588,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 257.992488125,
+ "first": 258.028639958,
+ "last": 259.839535875,
+ "chunks": 66
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 315.9810922490545,
+ "ttft_max": 0.15894150000002583,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 80.98834255321776,
+ "seconds": 3.148601292,
+ "prefill_seconds": 0.079248459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119086042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 259.842767708,
+ "first": 259.962342875,
+ "last": 263.11114075,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 79.98105127825862,
+ "seconds": 3.188255167,
+ "prefill_seconds": 0.079208459,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.079422916,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 259.842903583,
+ "first": 259.922682583,
+ "last": 263.110874791,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 82.01407486572037,
+ "seconds": 3.109222416,
+ "prefill_seconds": 0.118653292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.158542541,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 259.842822541,
+ "first": 260.001764041,
+ "last": 263.111057041,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 79.00105489643892,
+ "seconds": 3.227805,
+ "prefill_seconds": 0.039626042,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.039891209,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 259.842679041,
+ "first": 259.8830995,
+ "last": 263.110842166,
+ "chunks": 89
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 212.29582007221308,
+ "ttft_max": 0.03140174999998635,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 212.119392935892,
+ "seconds": 1.202153167,
+ "prefill_seconds": 0.030904875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.031109208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 43,
+ "drafted": 658,
+ "accepted": 213,
+ "acceptance_rate": 0.32370820668693007,
+ "tokens_per_round": 5.953488372093023
+ },
+ "sent": 267.675353583,
+ "first": 267.706755333,
+ "last": 268.907909458,
+ "chunks": 44
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 379.89684590178143,
+ "ttft_max": 0.16034129099995198,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 99.56640459004593,
+ "seconds": 2.561104833,
+ "prefill_seconds": 0.159479333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.159775416,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 268.911875375,
+ "first": 269.072216666,
+ "last": 271.633347125,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 97.98995228321633,
+ "seconds": 2.602307625,
+ "prefill_seconds": 0.118263,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.118519333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 268.911995541,
+ "first": 269.031000583,
+ "last": 271.633265916,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 94.97800686107192,
+ "seconds": 2.684832083,
+ "prefill_seconds": 0.035676625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.035969,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 268.911934375,
+ "first": 268.94840775,
+ "last": 271.633184708,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 96.45839907288453,
+ "seconds": 2.643626708,
+ "prefill_seconds": 0.076915875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.077211458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 268.91179475,
+ "first": 268.989663083,
+ "last": 271.633212541,
+ "chunks": 68
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 141.11723064905576,
+ "ttft_max": 0.03655066699997178,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 141.04172122135094,
+ "seconds": 1.807975667,
+ "prefill_seconds": 0.036057083,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.036274583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 975,
+ "accepted": 191,
+ "acceptance_rate": 0.19589743589743588,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 271.636266583,
+ "first": 271.67281725,
+ "last": 273.4798255,
+ "chunks": 66
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 314.2222738472332,
+ "ttft_max": 0.1598498339999992,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 80.52107452016692,
+ "seconds": 3.166872791,
+ "prefill_seconds": 0.119563166,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119831125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 273.483816583,
+ "first": 273.604093333,
+ "last": 276.770985458,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 81.54333064693361,
+ "seconds": 3.12717175,
+ "prefill_seconds": 0.118842625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.159505875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 273.483945416,
+ "first": 273.64379525,
+ "last": 276.771061,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 78.55952839607261,
+ "seconds": 3.245946166,
+ "prefill_seconds": 0.040431458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.040645833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 273.483870916,
+ "first": 273.524951208,
+ "last": 276.770851416,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 79.52676274383238,
+ "seconds": 3.20646775,
+ "prefill_seconds": 0.079922917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.080200708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 273.483737416,
+ "first": 273.564455,
+ "last": 276.770873291,
+ "chunks": 90
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 212.0302205426082,
+ "ttft_max": 0.03127620800000841,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 211.84478174379822,
+ "seconds": 1.2037115,
+ "prefill_seconds": 0.03071525,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.030954333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 43,
+ "drafted": 658,
+ "accepted": 213,
+ "acceptance_rate": 0.32370820668693007,
+ "tokens_per_round": 5.953488372093023
+ },
+ "sent": 281.280602625,
+ "first": 281.311878833,
+ "last": 282.514537583,
+ "chunks": 44
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 379.1363966258468,
+ "ttft_max": 0.16037083300000177,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 99.36423934604692,
+ "seconds": 2.566315625,
+ "prefill_seconds": 0.159496,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.159762459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 282.519196958,
+ "first": 282.679567791,
+ "last": 285.24584725,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 97.78525189785174,
+ "seconds": 2.607755209,
+ "prefill_seconds": 0.11804275,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.118341166,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 282.519314875,
+ "first": 282.638117083,
+ "last": 285.245836083,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 94.78495060310176,
+ "seconds": 2.6903005,
+ "prefill_seconds": 0.035455125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0357835,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 282.519259916,
+ "first": 282.555522583,
+ "last": 285.245777125,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 96.25564895196716,
+ "seconds": 2.649195167,
+ "prefill_seconds": 0.076599458,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.0768925,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 254,
+ "accepted": 188,
+ "acceptance_rate": 0.7401574803149606,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 282.51910575,
+ "first": 282.596677875,
+ "last": 285.245823958,
+ "chunks": 68
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 141.19561917656972,
+ "ttft_max": 0.03684129200001962,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 141.1207905160334,
+ "seconds": 1.806962667,
+ "prefill_seconds": 0.036193416,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03647625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 975,
+ "accepted": 191,
+ "acceptance_rate": 0.19589743589743588,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 285.249543708,
+ "first": 285.286385,
+ "last": 287.092390041,
+ "chunks": 66
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 314.095888200934,
+ "ttft_max": 0.1594045840000149,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 81.49345545633638,
+ "seconds": 3.129085625,
+ "prefill_seconds": 0.158699542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.158901208,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 287.097095166,
+ "first": 287.25649975,
+ "last": 290.385539375,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 79.4908501939986,
+ "seconds": 3.207916375,
+ "prefill_seconds": 0.079819542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.080064833,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 287.09728275,
+ "first": 287.177629958,
+ "last": 290.385496,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 78.52426526343883,
+ "seconds": 3.247403833,
+ "prefill_seconds": 0.040318292,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.040579875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 287.097213666,
+ "first": 287.138123416,
+ "last": 290.385478208,
+ "chunks": 90
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 80.47927459453535,
+ "seconds": 3.168517625,
+ "prefill_seconds": 0.119257542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119520917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 89,
+ "drafted": 278,
+ "accepted": 166,
+ "acceptance_rate": 0.5971223021582733,
+ "tokens_per_round": 2.8764044943820224
+ },
+ "sent": 287.097006458,
+ "first": 287.217065125,
+ "last": 290.385527291,
+ "chunks": 90
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 212.46021523124688,
+ "ttft_max": 0.031284874999983,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 212.2716090156747,
+ "seconds": 1.201291125,
+ "prefill_seconds": 0.03071125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.03097,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 43,
+ "drafted": 658,
+ "accepted": 213,
+ "acceptance_rate": 0.32370820668693007,
+ "tokens_per_round": 5.953488372093023
+ },
+ "sent": 294.944889416,
+ "first": 294.976174291,
+ "last": 296.176399,
+ "chunks": 44
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 379.1938092247709,
+ "ttft_max": 0.1560188750000293,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 94.80090021819542,
+ "seconds": 2.689847875,
+ "prefill_seconds": 0.031562542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.031844084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 1
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 251,
+ "accepted": 188,
+ "acceptance_rate": 0.749003984063745,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 296.181794916,
+ "first": 296.214057333,
+ "last": 298.903864,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 97.7997297917729,
+ "seconds": 2.607369167,
+ "prefill_seconds": 0.082535875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.114337917,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 255,
+ "accepted": 188,
+ "acceptance_rate": 0.7372549019607844,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 296.1819095,
+ "first": 296.296626625,
+ "last": 298.903974666,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 99.37142863907908,
+ "seconds": 2.566129958,
+ "prefill_seconds": 0.123787125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.155574042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 255,
+ "accepted": 188,
+ "acceptance_rate": 0.7372549019607844,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 296.181857875,
+ "first": 296.33787675,
+ "last": 298.903949958,
+ "chunks": 68
+ },
+ {
+ "usage": {
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "total_tokens": 270,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 96.2784902404724,
+ "seconds": 2.648566667,
+ "prefill_seconds": 0.041288375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.073060333,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "a10c9b848ed9",
+ "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130",
+ "min_rows": 4
+ },
+ "speculative": {
+ "rounds": 67,
+ "drafted": 255,
+ "accepted": 188,
+ "acceptance_rate": 0.7372549019607844,
+ "tokens_per_round": 3.8208955223880596
+ },
+ "sent": 296.181708541,
+ "first": 296.255363,
+ "last": 298.90387875,
+ "chunks": 68
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 1,
+ "aggregate_decode_tps": 141.31967120959874,
+ "ttft_max": 0.03666970799997671,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 141.24636394193516,
+ "seconds": 1.805356208,
+ "prefill_seconds": 0.036061958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.036302625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 16
+ },
+ "speculative": {
+ "rounds": 65,
+ "drafted": 975,
+ "accepted": 191,
+ "acceptance_rate": 0.19589743589743588,
+ "tokens_per_round": 3.9384615384615387
+ },
+ "sent": 298.908591208,
+ "first": 298.945260916,
+ "last": 300.749680625,
+ "chunks": 66
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "chat",
+ "streams": 4,
+ "aggregate_decode_tps": 316.7578775374469,
+ "ttft_max": 0.15931695800003354,
+ "requests": [
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 82.21321081821357,
+ "seconds": 3.101691291,
+ "prefill_seconds": 0.118398959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.158899334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 300.755190833,
+ "first": 300.914507791,
+ "last": 304.016232083,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 80.17450267878608,
+ "seconds": 3.180562292,
+ "prefill_seconds": 0.079673916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.079946708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 300.755306,
+ "first": 300.835610666,
+ "last": 304.016114666,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 81.17991883253512,
+ "seconds": 3.141170916,
+ "prefill_seconds": 0.119087542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.119339375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 300.755252541,
+ "first": 300.875013291,
+ "last": 304.016133875,
+ "chunks": 89
+ },
+ {
+ "usage": {
+ "prompt_tokens": 19,
+ "completion_tokens": 256,
+ "total_tokens": 275,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "qwen27",
+ "tokens_per_second": 79.19185080228227,
+ "seconds": 3.220028291,
+ "prefill_seconds": 0.040182916,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.040450042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "e2c694d862fd",
+ "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff",
+ "min_rows": 3
+ },
+ "speculative": {
+ "rounds": 88,
+ "drafted": 276,
+ "accepted": 167,
+ "acceptance_rate": 0.605072463768116,
+ "tokens_per_round": 2.909090909090909
+ },
+ "sent": 300.755108875,
+ "first": 300.796106958,
+ "last": 304.016091958,
+ "chunks": 89
+ }
+ ]
+ }
+ ],
+ "cold-prefill": [
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 9.475e-05,
+ "prefill_seconds": 0.152224375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.152786125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.15443183299999852
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.8958e-05,
+ "prefill_seconds": 0.602027959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.60268975,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.6041695000000011
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.6042e-05,
+ "prefill_seconds": 2.92371825,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.924125542,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 2.9251302499999987
+ },
+ {
+ "round": 0,
+ "arm": "main-plain",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.6083e-05,
+ "prefill_seconds": 7.367716917,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.368493834,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.370146667
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.9416e-05,
+ "prefill_seconds": 0.142128125,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.142321375,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.1428019169999999
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.4333e-05,
+ "prefill_seconds": 0.596952333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.597156042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.5976716660000001
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.4583e-05,
+ "prefill_seconds": 2.92241675,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.922817417,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 2.9238302500000017
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.5083e-05,
+ "prefill_seconds": 7.322124959,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.322879042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.324568999999997
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 5.0375e-05,
+ "prefill_seconds": 0.162896375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.16343125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.16452287500000295
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.7708e-05,
+ "prefill_seconds": 0.632174167,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.632618458,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.6336222919999983
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.8084e-05,
+ "prefill_seconds": 3.046362584,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 3.046803042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 3.0479570839999965
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.6958e-05,
+ "prefill_seconds": 7.564455625,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.565224041,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.566995167000002
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.5834e-05,
+ "prefill_seconds": 0.142068208,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.142249625,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.14270199999999988
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.5542e-05,
+ "prefill_seconds": 0.596721958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.596929459,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.5973893749999988
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.4916e-05,
+ "prefill_seconds": 2.922063542,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.922471709,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 2.923471584000005
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.1833e-05,
+ "prefill_seconds": 7.322574708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.323297667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.325058874999996
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 4.8791e-05,
+ "prefill_seconds": 0.16274975,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.163254084,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.16436254200000633
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 2.5375e-05,
+ "prefill_seconds": 0.632193375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.632682667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.6336696670000066
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.8917e-05,
+ "prefill_seconds": 3.045708417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 3.046133125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 3.0472295000000003
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 4.0959e-05,
+ "prefill_seconds": 7.564470417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.565208167,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.566989333999999
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 6.125e-05,
+ "prefill_seconds": 0.152161958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.152673334,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.15385920900000372
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.5958e-05,
+ "prefill_seconds": 0.60266275,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.603419666,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.6049087920000034
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.7e-05,
+ "prefill_seconds": 2.924098958,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.924542125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 2.9256166250000035
+ },
+ {
+ "round": 1,
+ "arm": "main-plain",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.725e-05,
+ "prefill_seconds": 7.326498708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.32727875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.329111124999997
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 6.3167e-05,
+ "prefill_seconds": 0.16316275,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.163665083,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.16489629199999456
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 4.0083e-05,
+ "prefill_seconds": 0.632195333,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.632729583,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.633867084000002
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 4.2125e-05,
+ "prefill_seconds": 3.046123,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 3.04655725,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 3.0476708340000016
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 4.1291e-05,
+ "prefill_seconds": 7.564734041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.565487958,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.567298958999999
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.9333e-05,
+ "prefill_seconds": 0.142121708,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.142318875,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.14281366699999865
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.575e-05,
+ "prefill_seconds": 0.597101375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.597292042,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.5977911669999969
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.6833e-05,
+ "prefill_seconds": 2.923957417,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.924390292,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 2.9254025830000074
+ },
+ {
+ "round": 2,
+ "arm": "main-plain",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.6583e-05,
+ "prefill_seconds": 7.32684075,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.327646125,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.329436958000002
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt_tokens": 2048,
+ "usage": {
+ "prompt_tokens": 2048,
+ "completion_tokens": 1,
+ "total_tokens": 2049,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.7417e-05,
+ "prefill_seconds": 0.142097667,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.14230825,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.1427672080000093
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt_tokens": 8192,
+ "usage": {
+ "prompt_tokens": 8192,
+ "completion_tokens": 1,
+ "total_tokens": 8193,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.4916e-05,
+ "prefill_seconds": 0.596914375,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 0.597105708,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "94c47d91f1be",
+ "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 0.5975776660000065
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt_tokens": 32768,
+ "usage": {
+ "prompt_tokens": 32768,
+ "completion_tokens": 1,
+ "total_tokens": 32769,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 3.4459e-05,
+ "prefill_seconds": 2.922614041,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 2.923045667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "f0cbba2470c7",
+ "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 2.9240610420000053
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt_tokens": 65536,
+ "usage": {
+ "prompt_tokens": 65536,
+ "completion_tokens": 1,
+ "total_tokens": 65537,
+ "prompt_tokens_details": {
+ "cached_tokens": 0
+ },
+ "completion_tokens_details": {
+ "reasoning_tokens": 0
+ }
+ },
+ "runtime": {
+ "enable_thinking": false,
+ "reasoning_effort": "none",
+ "engine": "lanes",
+ "tokens_per_second": 0.0,
+ "seconds": 2.925e-05,
+ "prefill_seconds": 7.347452875,
+ "prefill_widths": [],
+ "prefill_raised": [],
+ "time_to_first_token": 7.348175667,
+ "sampling": "greedy",
+ "drafts": true,
+ "token_sha": "3635a91e3da8",
+ "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0",
+ "min_rows": 0
+ },
+ "speculative": {
+ "rounds": 0,
+ "drafted": 0,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 0.0
+ },
+ "wall": 7.349954542000006
+ }
+ ]
+ }
+ }
+}
diff --git a/docs/receipts/lfm2-dspark-review/performance-investigation.json b/docs/receipts/lfm2-dspark-review/performance-investigation.json
new file mode 100644
index 0000000000..6bc04e9b0a
--- /dev/null
+++ b/docs/receipts/lfm2-dspark-review/performance-investigation.json
@@ -0,0 +1,1517 @@
+{
+ "status": "M4 and M5 comparison complete; no no-loss pass against main",
+ "published_source": "8bd05874d98d9449ad2f8ddb8796bf26e229874b",
+ "device": "Apple M4 Pro, 20 GPU cores, 48 GiB",
+ "protocol": {
+ "generated_tokens": 256,
+ "warmup_tokens": 32,
+ "context": 4096,
+ "parallel_limit": 16,
+ "cache": 0,
+ "sampling": "greedy, seed 1234",
+ "aggregate_tps": "streams * 255 / (latest last token - earliest first token)",
+ "arm_order_by_round": [
+ [
+ "published",
+ "candidate"
+ ],
+ [
+ "candidate",
+ "published"
+ ],
+ [
+ "published",
+ "candidate"
+ ],
+ [
+ "published",
+ "candidate"
+ ]
+ ]
+ },
+ "source_sha256": {
+ "zig/src/families/lfm2_moe/dspark_block.zig": "d2dd077581904759967446ebe2ad195c083ae73c004862de09907ffb4671ac41",
+ "zig/src/families/lfm2_moe/dspark.zig": "06b5dda907dcb6f44804edfd0c66466bb7422365516613478c777a6e55f46302",
+ "zig/src/core/lanes/hold_price.zig": "2e6cbe4c477ae19c9edc527a74471748fc8af012725f402ee4c81ed3bf021dba"
+ },
+ "cells": [
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 135.60663824374626,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 237.5475197924822,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 270.9110025112889,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 325.10385752712824,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 148.10711121092461,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 270.0303059152453,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 295.6105956217913,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 322.43096510198524,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 43.99498661294338,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 258.03903047977514,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 287.71439774595365,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 329.26775089681786,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 155.17129240785718,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 274.42316268438344,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 293.0455742669573,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 324.3134795211021,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 137.1292625523315,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 252.08277989639137,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 284.07929135629126,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 327.5277632360025,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 150.82092426693282,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 268.80377082796355,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 294.66891749695225,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 323.9697195614469,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 136.14276531242143,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 252.69384755422968,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 284.64110269593544,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 325.6922837054757,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 150.98369206524345,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 268.60609628493586,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 293.288959146046,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 323.24421841141884,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 136.28826488314226,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 251.47134877024175,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 284.5448538421833,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 325.79926741762347,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 150.90184937588137,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 234.90340221222957,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 293.26972222916555,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 322.47025493738624,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 138.34750502292397,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 254.00120377530746,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 284.5375280962798,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 327.2831823875495,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 151.00505713344813,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 271.8840625705605,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 298.09945887703077,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 324.2089086579436,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 136.32707890068104,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 251.8205867417788,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 284.349538989573,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 325.9626710889029,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 151.20030809404938,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 268.07908129520496,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 293.3722674786694,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 323.0746966258725,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 3,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 138.18860967197506,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 3,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 251.84917011393054,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 3,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 284.3401360103994,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 3,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 327.6480036147952,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 3,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 151.6843393961539,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 3,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 268.8005329779927,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 3,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 294.4169012094223,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 3,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 324.05536974614046,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ }
+ ],
+ "limitations": [
+ "One candidate lone-code cell was 43.99 tok/s; retained in the four-round median. No competing model process was observed when subsequently checked.",
+ "Four rounds were not perfectly balanced: the final order repeated published then candidate.",
+ "Earlier kernel/context candidate regressed M4 16-stream prose by 8.38%; the retained version preserves original context-ingestion arithmetic.",
+ "An acceptance-cap scheduler experiment lost 2\u20133% on some M5 workloads and is excluded."
+ ],
+ "candidate_source": "b4deee277",
+ "m5": {
+ "device": "Apple M5 Ultra, 256 GiB; macOS 27.0 (26A428); Zig 0.17.0; Metal 32023.921.6",
+ "source_sha256": {
+ "zig/src/core/lanes/windows.zig": "88eedd02202e42e91fcdef4dd046d969038cc391cac282604893a417af3ad13a",
+ "zig/src/core/lanes/stream.zig": "7a129645b82569a0f9b680088fe35f5e9bd1defb9149e14a8fa85f777ebf48d0",
+ "zig/src/families/lfm2_moe/dspark_block.zig": "d2dd077581904759967446ebe2ad195c083ae73c004862de09907ffb4671ac41",
+ "zig/tests/contract/metal_lfm2_bf16.zig": "06f08a81d48a17371bdaed3b1c036aee085af7d7c0e0eed258b845e51a85c774"
+ },
+ "cells": [
+ {
+ "round": 0,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 289.99017871497716,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 651.5857918743773,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 831.2316832063025,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 831.1221850446695,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 261.653824038573,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 622.0862892391041,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 804.3648359431719,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 799.608159390429,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 354.34974736947413,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 645.481121931619,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 810.5269440440464,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 797.0694737428038,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 356.3219816253091,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 618.0402465118209,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 789.0806197623405,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 791.0577258755956,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 354.7364418761545,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 646.0307516882782,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 815.7156281542954,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 798.470003561415,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 356.5780465065517,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 618.6789173179606,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 796.0023513285163,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 792.391417523257,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 356.9031926500828,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 645.8077958032181,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 813.5162265998285,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 796.4582658126303,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 356.61902076187044,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 618.9625704407207,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 794.9411091902847,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 790.0706195599455,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 351.7006307459781,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 644.9662400483767,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 813.0724108239679,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 801.6591921437237,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 355.0560737481366,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 618.0832683633394,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 795.426957848166,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 796.7172061115025,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 289.0384163078719,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 652.2329226077759,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 831.6711661442548,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 830.8481776811126,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 262.70151064092926,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 622.7723581572978,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 805.1425054893175,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 800.4781222863546,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 353.41984243522757,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 645.7325520628694,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 810.1592109631267,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 804.1400599123751,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 356.47342949874684,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 618.7608129248248,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 789.0114172609233,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "candidate",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 798.1338794206057,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 290.3072664135369,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 651.6165425845753,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 831.5210047708392,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "main-no-external-drafter",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 829.3651009986223,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 261.76873963295424,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 622.0101018709207,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 804.9815598809657,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "main-no-external-drafter",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 798.5681100696453,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "aggregate_decode_tps": 353.57965911849703,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "aggregate_decode_tps": 645.0730230346089,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "aggregate_decode_tps": 809.4871362230531,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "aggregate_decode_tps": 797.7172187774961,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "aggregate_decode_tps": 356.74670092367637,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "aggregate_decode_tps": 618.772261173814,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "aggregate_decode_tps": 797.1883738155475,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "aggregate_decode_tps": 791.4675338150536,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "binary_sha256": {
+ "zig-out/native/bin/tensorfold-native": "d5c4358dc174587536ab242aa438fb9fea9557df7a719bef83c35dd9397127c9",
+ "zig-out/bin/tf-lfm2-dspark": "29519d0208e77721fab62fbf63d982ec8db82a302b603999e185b462b19eb058",
+ "zig-out/bin/tf-kernel-contract": "4fd08e7265bcec265b92a682f3fe927c7580acf6a1585b69daccfde0936ef6ad"
+ },
+ "protocol": {
+ "rounds": 3,
+ "arm_order_by_round": [
+ [
+ "main",
+ "published",
+ "candidate"
+ ],
+ [
+ "published",
+ "candidate",
+ "main"
+ ],
+ [
+ "candidate",
+ "main",
+ "published"
+ ]
+ ],
+ "same_requests_as_m4": true,
+ "all_72_cells_match": true
+ },
+ "qualification": {
+ "source": "b4deee277",
+ "stress": [
+ {
+ "slots": 6,
+ "seconds": 60,
+ "completed": 423,
+ "cancelled": 234,
+ "wrong": 0,
+ "errors": []
+ },
+ {
+ "slots": 16,
+ "seconds": 90,
+ "completed": 827,
+ "cancelled": 445,
+ "wrong": 0,
+ "errors": []
+ }
+ ],
+ "sampled": [
+ {
+ "streams": 1,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ]
+ ]
+ },
+ {
+ "streams": 4,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ]
+ ]
+ },
+ {
+ "streams": 8,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ],
+ [
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366"
+ ],
+ [
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff"
+ ],
+ [
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31"
+ ],
+ [
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf"
+ ]
+ ]
+ },
+ {
+ "streams": 16,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ],
+ [
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366"
+ ],
+ [
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff"
+ ],
+ [
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31"
+ ],
+ [
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf"
+ ],
+ [
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986",
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986",
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986"
+ ],
+ [
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351",
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351",
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351"
+ ],
+ [
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369",
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369",
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369"
+ ],
+ [
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a",
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a",
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a"
+ ],
+ [
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363",
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363",
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363"
+ ],
+ [
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398",
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398",
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398"
+ ],
+ [
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db",
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db",
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db"
+ ],
+ [
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd",
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd",
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd"
+ ]
+ ]
+ }
+ ],
+ "resume": {
+ "equal": true,
+ "cached_tokens": 3517,
+ "resumed_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91",
+ "fresh_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91"
+ },
+ "8bit": [
+ {
+ "streams": 1,
+ "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "concurrent_sha256": [
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58"
+ ]
+ },
+ {
+ "streams": 4,
+ "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "concurrent_sha256": [
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58"
+ ]
+ },
+ {
+ "streams": 8,
+ "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "concurrent_sha256": [
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58"
+ ]
+ },
+ {
+ "streams": 16,
+ "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "concurrent_sha256": [
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58"
+ ]
+ },
+ {
+ "streams": 1,
+ "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "concurrent_sha256": [
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b"
+ ]
+ },
+ {
+ "streams": 4,
+ "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "concurrent_sha256": [
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b"
+ ]
+ },
+ {
+ "streams": 8,
+ "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "concurrent_sha256": [
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b"
+ ]
+ },
+ {
+ "streams": 16,
+ "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "concurrent_sha256": [
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b"
+ ]
+ }
+ ],
+ "long_context": [
+ {
+ "length": 65536,
+ "plain_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "solo_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "concurrent_sha256": [
+ "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6"
+ ]
+ },
+ {
+ "length": 130816,
+ "plain_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "solo_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "concurrent_sha256": [
+ "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3"
+ ]
+ }
+ ],
+ "kernel_contract": "11 pass, 0 fail; 17 controls red, 0 missed; 0 limit problems",
+ "drafter_reference": [
+ "step 6: confidence reference 0.1449, GPU 0.1465",
+ "step 7: reference 496 (margin 1.722, second 55658), GPU 496 equal",
+ "step 7: confidence reference 0.1030, GPU 0.1027",
+ "step 8: reference 49 (margin 0.100, second 28358), GPU 49 equal",
+ "step 8: confidence reference 0.0880, GPU 0.0873",
+ "the reference's largest margin over a different GPU pick: 0.000",
+ "the confidences' worst difference from the fp32 reference: 0.00158",
+ "9 of 9 drafts equal the fp32 reference's picks on the same chain (255-token context)"
+ ]
+ }
+ }
+}
diff --git a/docs/receipts/lfm2-dspark-review/qualification.py b/docs/receipts/lfm2-dspark-review/qualification.py
new file mode 100644
index 0000000000..e0987af059
--- /dev/null
+++ b/docs/receipts/lfm2-dspark-review/qualification.py
@@ -0,0 +1,225 @@
+import concurrent.futures as cf
+import fcntl
+import json
+import os
+from pathlib import Path
+import random
+import signal
+import statistics
+import subprocess
+import sys
+import threading
+import time
+import urllib.request
+
+ROOT = Path(__file__).resolve().parents[3]
+OUT = Path(os.environ.get('TF_QUAL_OUT', '/tmp/tf1394-evidence'))
+OUT.mkdir(exist_ok=True)
+HUB = Path.home()/'.cache/huggingface/hub'
+MODEL = Path(os.environ.get('TF_QUAL_MODEL', HUB/'models--LiquidAI--LFM2.5-24B-A2B-MLX-4bit/snapshots/667408d494bde191c2cb9dd4f4549827816b9275'))
+DRAFT = Path(os.environ.get('TF_QUAL_DRAFTER', Path.home()/'models/LiquidAI--LFM2.5-24B-A2B-DSpark'))
+CAND = Path(os.environ.get('TF_QUAL_CANDIDATE', ROOT/'zig-out/native/bin/tensorfold-native'))
+MAIN = Path(os.environ.get('TF_QUAL_MAIN', '/tmp/tf1394-main-base/zig-out/native/bin/tensorfold-native'))
+BASE = 'http://127.0.0.1:18194'
+PROMPTS = json.loads((ROOT/'tools/bench_prompts.json').read_text())[:8]
+
+def emit(kind, **data):
+ print(json.dumps({'event':kind, **data}), flush=True)
+
+def save(name, data):
+ (OUT/(name+'.json')).write_text(json.dumps(data, indent=2)+'\n')
+
+def post(body, stream=False, cancel=None):
+ body = {'model':'review', 'max_tokens':128, 'temperature':0, 'seed':1234,
+ 'ignore_eos':True, 'return_token_ids':True, **body, 'stream':stream}
+ if stream: body['stream_options']={'include_usage':True}
+ chat='messages' in body
+ if chat: body['chat_template_kwargs']={'enable_thinking':False}
+ endpoint='/v1/chat/completions' if chat else '/v1/completions'
+ req = urllib.request.Request(BASE+endpoint, data=json.dumps(body).encode(), headers={'Content-Type':'application/json'})
+ sent = time.perf_counter()
+ with urllib.request.urlopen(req, timeout=3600) as r:
+ if not stream:
+ x=json.load(r)
+ content=x['choices'][0].get('message',{}).get('content',x['choices'][0].get('text',''))
+ return {'usage':x['usage'], 'runtime':x['tensorfold'], 'speculative':x.get('speculative',{}), 'content':content, 'wall':time.perf_counter()-sent}
+ usage={}; runtime={}; spec={}; first=None; last=None; chunks=0; done=False
+ for raw in r:
+ line=raw.decode().strip()
+ if line=='data: [DONE]': done=True; break
+ if not line.startswith('data:'): continue
+ x=json.loads(line[5:])
+ if 'error' in x: raise RuntimeError(str(x['error']))
+ usage=x.get('usage') or usage; runtime=x.get('tensorfold') or runtime; spec=x.get('speculative') or spec
+ if any(c.get('text') or c.get('delta',{}).get('content') for c in x.get('choices',[])):
+ now=time.perf_counter(); first=first or now; last=now; chunks+=1
+ if cancel and chunks>=cancel: return {'cancelled':True}
+ if not done or not runtime.get('token_sha256'): raise RuntimeError('incomplete stream')
+ return {'usage':usage, 'runtime':runtime, 'speculative':spec, 'sent':sent, 'first':first, 'last':last, 'chunks':chunks}
+
+class Server:
+ def __init__(self, label, binary=CAND, mode='bf16', context=16384, parallel=16, cache=0, validation=False, model=MODEL, draft=DRAFT):
+ self.label=label; self.binary=binary; self.mode=mode; self.context=context; self.parallel=parallel
+ self.cache=cache; self.validation=validation; self.model=model; self.draft=draft
+ def __enter__(self):
+ processes=subprocess.check_output(['ps','-Ao','pid=,comm='],text=True)
+ busy=[line for line in processes.splitlines() if line.split() and Path(line.split()[-1]).name in ('tensorfold-native','tf-lfm2-exact','tf-lfm2-dspark')]
+ if busy: raise RuntimeError('another model process is already running')
+ args=[str(self.binary),'serve',str(self.model),'--name','review','--host','127.0.0.1','--port','18194','--parallel',str(self.parallel),'--context',str(self.context),'--prompt-cache-gib',str(self.cache),'--keep-warm','900']
+ if self.mode!='plain': args+=['--drafter',str(self.draft)]
+ if self.mode=='8': args+=['--drafter-bits','8']
+ env=dict(os.environ)
+ if self.validation: env['MTL_DEBUG_LAYER']='1'
+ self.log=open(OUT/(self.label+'.server.log'),'w')
+ self.proc=subprocess.Popen(args, stdout=self.log, stderr=subprocess.STDOUT, env=env, start_new_session=True)
+ start=time.monotonic()
+ while time.monotonic()-start<600:
+ if self.proc.poll() is not None: raise RuntimeError(self.label+' server exited')
+ try:
+ with urllib.request.urlopen(BASE+'/v1/models',timeout=1) as r: json.load(r)
+ emit('ready', label=self.label, seconds=time.monotonic()-start)
+ return self
+ except Exception: time.sleep(.5)
+ self.__exit__(None,None,None)
+ raise TimeoutError('load timeout')
+ def __exit__(self,*args):
+ started=time.monotonic(); forced=False
+ if self.proc.poll() is None:
+ self.proc.send_signal(signal.SIGTERM)
+ try: self.proc.wait(timeout=90)
+ except subprocess.TimeoutExpired:
+ forced=True; os.killpg(self.proc.pid,signal.SIGKILL); self.proc.wait()
+ self.log.close()
+ save(self.label+'.shutdown',{'forced_kill':forced,'seconds':time.monotonic()-started,'returncode':self.proc.returncode})
+ if forced: raise RuntimeError(self.label+' required forced shutdown')
+
+def digest(r): return r['runtime']['token_sha256']
+
+def functional():
+ results={}
+ first_messages=[{'role':'user','content':'Public cache fixture.\n'+('alpha beta gamma delta\n'*700)+'\nExplain matrix multiplication in one paragraph.'}]
+ with Server('functional-cache',cache=1,validation=True):
+ first=post({'messages':first_messages,'max_tokens':64})
+ messages=first_messages+[{'role':'assistant','content':first['content']},{'role':'user','content':'Give a small numerical example and explain each step.'}]
+ resumed=post({'messages':messages,'max_tokens':64})
+ if resumed['usage'].get('prompt_tokens_details',{}).get('cached_tokens',resumed['usage'].get('cached_tokens',0))<=0:
+ raise AssertionError('resume did not report reused tokens: '+str(resumed['usage']))
+ results['resumed']=resumed; results['resume_request']={'messages':messages,'max_tokens':64}
+ refs=[post({'prompt':p['prompt'],'max_tokens':64}) for p in PROMPTS]
+ results['stress_reference_hashes']=[{'prompt':p['name'],'sha256':digest(r)} for p,r in zip(PROMPTS,refs)]
+ results['stress']=[]
+ for slots,seconds in [(6,60),(16,90)]:
+ end=time.monotonic()+seconds
+ def worker(i):
+ rng=random.Random(4242+i); counts={'completed':0,'cancelled':0,'wrong':0,'errors':[]}
+ while time.monotonic()0
+ sampled=[]
+ for n in [1,4,8,16]:
+ bodies=[{'prompt':PROMPTS[i%len(PROMPTS)]['prompt'],'temperature':.7,'top_p':.95,'seed':4242+i,'max_tokens':64} for i in range(n)]
+ plain=[post({**b,'draft':False}) for b in bodies]
+ solo=[post(b) for b in bodies]
+ gate=threading.Barrier(n)
+ def run(b): gate.wait(); return post(b)
+ with cf.ThreadPoolExecutor(n) as ex: together=list(ex.map(run,bodies))
+ row={'streams':n,'hashes':[[digest(p),digest(s),digest(t)] for p,s,t in zip(plain,solo,together)]}
+ sampled.append(row); emit('sampled',streams=n,equal=sum(len(set(x))==1 for x in row['hashes']))
+ assert all(len(set(x))==1 for x in row['hashes'])
+ results['sampled']=sampled; save('functional',results)
+ with Server('functional-fresh'):
+ fresh=post({'messages':messages,'max_tokens':64})
+ assert digest(fresh)==digest(results['resumed'])
+ results['fresh']=fresh; results['resume_equal']=True; save('functional',results); emit('resume',equal=True)
+
+def long_context():
+ results=[]
+ with Server('long-context',parallel=1,context=131072):
+ tail=[825,264,15352,4829,9944,13,271,279,854,22373]
+ for length in [4096,16384,65536,130816]:
+ ids=(tail*((length+9)//10))[:length]
+ plain=post({'prompt':ids,'max_tokens':64,'draft':False})
+ drafted=post({'prompt':ids,'max_tokens':64})
+ row={'length':length,'plain':plain,'drafted':drafted,'equal':digest(plain)==digest(drafted)}
+ results.append(row); save('long-context',results); emit('long-context',length=length,equal=row['equal'])
+ assert row['equal']
+ for length in [65536,130816]:
+ emit('long-reference-start',length=length)
+ with open(OUT/('dspark-reference-'+str(length)+'.log'),'w') as log:
+ r=subprocess.run([str(ROOT/'zig-out/bin/tf-lfm2-dspark'),str(MODEL),str(DRAFT),str(length)],stdout=log,stderr=subprocess.STDOUT,timeout=3600)
+ emit('long-reference-done',length=length,exit=r.returncode)
+ assert r.returncode==0
+
+def performance(family='lfm2', compare=False):
+ model=MODEL; draft=DRAFT; widths=[1,4,8,16]
+ if family=='qwen':
+ model=HUB/'models--TensorFold--Qwen3.8-27B-MLX-4bit/snapshots/22d8d538154e0e5b6f8dcb1fcc74a60a0798104c'
+ draft=HUB/'models--z-lab--Qwen3.8-27B-DFlash2/snapshots/50307d4c4cde6860d4eee73e2547cd786fe8e8a4'
+ widths=[1,4]
+ prompts=[{'name':'code','prompt':'Write a short Python function that computes the Fibonacci sequence and explain it.'},{'name':'chat','prompt':'Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.'}]
+ arms={'main-plain':(MAIN,'plain'),'candidate-plain':(CAND,'plain'),'candidate-drafted':(CAND,'bf16')}
+ if family=='qwen': arms={'main-drafted':(MAIN,'bf16'),'candidate-drafted':(CAND,'bf16')}
+ if compare:
+ published=os.environ.get('TF_QUAL_PUBLISHED')
+ if not published: raise SystemExit('compare requires TF_QUAL_PUBLISHED pointing to the earlier PR binary')
+ arms={'main':(MAIN,'plain'),'published':(Path(published),'bf16'),'candidate':(CAND,'bf16')}
+ results=[]; reference={}
+ order=list(arms)
+ for round_ in range(3):
+ rotated=order[round_%len(order):]+order[:round_%len(order)]
+ for arm in rotated:
+ binary,mode=arms[arm]
+ with Server(f'perf-{family}-{round_}-{arm}',binary,mode,4096,max(widths),0,False,model,draft):
+ for p in prompts: post({'prompt':p['prompt'],'max_tokens':32})
+ for p in prompts:
+ for width in widths:
+ gate=threading.Barrier(width)
+ def run(i): gate.wait(); return post({'prompt':p['prompt'],'max_tokens':256},True)
+ with cf.ThreadPoolExecutor(width) as ex: rows=list(ex.map(run,range(width)))
+ key=p['name']; hashes=[digest(x) for x in rows]
+ if key not in reference: reference[key]=hashes[0]
+ assert all(h==reference[key] for h in hashes),(family,arm,key,'token mismatch')
+ first=min(x['first'] for x in rows); last=max(x['last'] for x in rows)
+ row={'round':round_,'arm':arm,'prompt':key,'streams':width,'aggregate_decode_tps':width*255/(last-first),'ttft_max':max(x['first']-x['sent'] for x in rows),'requests':rows}
+ results.append(row); save(('comparison-' if compare else 'performance-')+family,results)
+ emit('performance',family=family,round=round_,arm=arm,prompt=key,streams=width,tps=round(row['aggregate_decode_tps'],2))
+
+def cold_prefill():
+ results=[]
+ arms={'main-plain':(MAIN,'plain'),'candidate-plain':(CAND,'plain'),'candidate-drafted':(CAND,'bf16')}
+ order=list(arms); cycle=[825,264,15352,4829,9944,13,271,279,854,22373]
+ for round_ in range(3):
+ for arm in order[round_:]+order[:round_]:
+ binary,mode=arms[arm]
+ with Server(f'cold-{round_}-{arm}',binary,mode,131072,1):
+ for length in [2048,8192,32768,65536]:
+ ids=(cycle*((length+9)//10))[:length]
+ r=post({'prompt':ids,'max_tokens':1})
+ assert r['usage']['prompt_tokens']==length
+ assert r['usage']['prompt_tokens_details']['cached_tokens']==0
+ row={'round':round_,'arm':arm,'prompt_tokens':length,**r}
+ results.append(row); save('cold-prefill',results)
+ emit('cold-prefill',round=round_,arm=arm,length=length,ttft=r['runtime']['time_to_first_token'])
+
+if __name__=='__main__':
+ phase=sys.argv[1]
+ with open('/tmp/tensorfold-model.lock','a') as lock:
+ emit('waiting-for-model-lock',phase=phase)
+ fcntl.flock(lock,fcntl.LOCK_EX)
+ emit('starting',phase=phase)
+ if phase=='functional': functional()
+ elif phase=='long': long_context()
+ elif phase=='performance': performance(sys.argv[2] if len(sys.argv)>2 else 'lfm2')
+ elif phase=='compare': performance(compare=True)
+ elif phase=='cold': cold_prefill()
diff --git a/docs/receipts/lfm2-dspark-review/wide-calibration-ablations.json b/docs/receipts/lfm2-dspark-review/wide-calibration-ablations.json
new file mode 100644
index 0000000000..020ae3fc71
--- /dev/null
+++ b/docs/receipts/lfm2-dspark-review/wide-calibration-ablations.json
@@ -0,0 +1,6480 @@
+{
+ "status": "diagnostic ablations; not the final qualification",
+ "base_source": "b4deee277",
+ "cells": {
+ "hold-accounting": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 643.8116172492632,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.867035866988,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.5423014255949,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5128631346989,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.2133939448045,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.4367288813849,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.8721684421929,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.7932585902063,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.4562456021381,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.8137429002606,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 789.3984943094403,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.8443467942008,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.5811969949731,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 807.0467578834366,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 795.2596029922896,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 621.5561777345076,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 793.5782789844875,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.4435265630628,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.5340653194621,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.5585853396626,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.2073874838301,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.1406888714015,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 789.2432337384452,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.516776880887,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.4465736255986,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 816.2421107964317,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 801.4234367658562,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.2008415099642,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.1245903763303,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 794.8434530980279,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.6861008016517,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.9269786391927,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.755833324863,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.0226926953744,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 789.1412486868718,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.3587036469579,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.835932459204,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 812.0914984701523,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.7322856971059,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.7651138042481,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 793.82458947546,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.9645002134771,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.7695329258135,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.4545693919375,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 800.0092092040525,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.6754308114879,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.5291124697678,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.4332636301492,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "overhead-isolation": [
+ {
+ "round": 0,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 651.9139356848392,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 833.2857206535231,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 835.9149562840696,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 622.5227827317697,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 805.6569359731286,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 804.9583461183761,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 626.6944335227236,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 825.5618638981348,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 807.9078668331922,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 623.0965393401463,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 819.1073107777997,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 801.4843109125941,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 626.0999271344892,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 825.3660475662691,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 808.7132776040836,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 622.8070889888528,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 819.3872788376917,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 802.0719142132929,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.4951725611897,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.6689741091631,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.5743750044154,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.2681572690839,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.5278723676245,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 791.9102164944356,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 626.7752548814751,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 826.4489176329733,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 814.3368120131368,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 624.0155122915002,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 819.9828479123096,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 807.0855718239884,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 627.5720069660566,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 826.5574111735665,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 807.7942775788831,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 623.9850367213689,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 820.0032146939689,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 799.4948745920863,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 643.34881150891,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.2084276073556,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.9371330451237,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.4374389899364,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 797.593366173878,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.8016607458413,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 652.8149539538148,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 832.4678228630254,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 834.935003851258,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 623.4896993541838,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 805.5968445877305,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 806.6520853665469,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 626.7696387571307,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 825.1367948484931,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "attached-oracle",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 811.6072972447571,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 622.8303818140437,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 818.3383747369702,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "attached-oracle",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 804.5607618463806,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.1967607125431,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.2916555717073,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 802.4850089088366,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.8951732623875,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.0171013259228,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 796.1992414445897,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 652.174121539032,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 831.2944461737667,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 830.5753163876966,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 621.904087145793,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 804.5647612762093,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 799.7848749620825,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 627.0169140117872,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 826.5210064063601,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "oracle",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 807.9543369033614,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 623.4856976506851,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 819.3914477642772,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "oracle",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 800.6927365909038,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "stage512": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.1002045152331,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.9288441624107,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.8704065651071,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.3453986463694,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.8361178213568,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.4605239532948,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 647.0598716246051,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.761762946578,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 797.2266217842038,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.7187047950405,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 793.6964676341051,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 791.9282262125578,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.6156236352848,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.4996930937969,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 796.9250218318138,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.1732304780877,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.2503544715339,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 790.6492548130739,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.3248762331575,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 819.4069301341176,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 800.3348459760032,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.1482365327016,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 797.5113084161665,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 794.1212643120977,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.1309082097619,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.3449806490839,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 805.1574545463609,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.9347379131115,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 789.7694274931367,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 798.4068000516954,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.753663704301,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.5240391620009,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.9495509997312,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 620.3189108172958,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.7201955248429,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.3067276189817,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.4792771524011,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.5435304341598,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.3027979206439,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 620.4576444123596,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.7101729324681,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.2221391903555,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.5922266917154,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 815.1055165738466,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.7626064655209,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5919007482223,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.3130704836641,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.7674920479974,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "phase-profile": [
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.7582181726698,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 803.2588749638213,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.8194277588038,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 796.3293510847767,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "copy-depth": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.5698231050824,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.7629997636277,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 800.0639724877383,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.9960916291277,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.9623119914172,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 794.7418540342475,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 635.8934085469378,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 807.7355727142193,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.4841326720517,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.4876546058689,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.8177306537868,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.8294791877873,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 638.8025415387899,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 807.1041519517509,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 800.2587895703317,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 620.0329453289825,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.0620024133443,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.5038779934085,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.2277789842284,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 812.2673339855877,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.2146474955022,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.8763193119653,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 791.5135137810487,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.9955237163615,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.5560253073638,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.1310537239268,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.7291201709987,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.7886021016695,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.1646976219921,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.1174413121428,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 638.2320557580033,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 807.8213740625819,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 802.7977304140775,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 620.561499278725,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.2709574286852,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 796.4268285351809,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 638.227196466546,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 807.7983826312196,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.0928664875843,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 620.9553872994748,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.0486200617715,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.4298028030543,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.2620333699359,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.6097290154547,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.0296424216548,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.5224101622193,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 792.2134225010636,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 796.9240813401055,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "shared-calibration": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 643.8672263414753,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.9204466072154,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 802.9999395980669,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.4098645026263,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.4934765134461,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 796.519437923307,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 632.7546042477178,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 812.8668321079678,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 883.5558321243161,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.2815390808294,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 794.8857673170639,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 864.9476620371097,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 642.7653023631718,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.3024790027683,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 886.3494767826272,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.9106785368859,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.1002468003629,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 863.7629682596074,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.8423531708951,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.731652812001,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.9885752219824,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.1531583535735,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.1099619610673,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.8953852759796,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.5601085995847,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 816.3072854901848,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.0876756467267,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 620.0547751768422,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 797.6908152230244,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.9558939987153,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 631.9349732717102,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 818.4119552954095,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 886.1439262176706,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.5384379760573,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 794.5709769513013,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 858.454954141176,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 632.4218011605403,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.1906073092416,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 893.9342616519325,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.1052974606797,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.9776983826855,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 863.5671192631019,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.0327904885318,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.5371187032524,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 801.2341625038504,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.3005940883706,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.3631061944923,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 794.7691593006111,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "shared-marginals": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.5886231883915,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.4363968702983,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 815.142402372856,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.5113247260725,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.2809506501271,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.7709316101826,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.6648510154018,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.1585280677888,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.7739736251266,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 634.0287094589685,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 816.4624306041238,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 884.521164233605,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.8545247109968,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.1133419790036,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.799379381399,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 873.9905629905045,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 352.80406289491833,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 640.5591704758675,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.1867558931708,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 884.3521287512664,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.4347333932498,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.0818501751139,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 794.3131848396016,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 855.1736597157891,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.6546246354652,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.2599910212919,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.5617690125395,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.9874073825234,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.3302798758094,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.2849439885823,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.289525016227,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.6706139105225,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.80283106837254,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.0808791826066,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 817.6687392464844,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 803.463096563945,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.94066175605167,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.753712363033,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.8178065343876,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 796.6235333344594,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.1044925522018,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 641.4679975502607,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 817.0524999042318,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 883.7644908150517,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.67978776377765,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.7057736525313,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.9200860651677,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 866.0757645413831,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 356.73545076881857,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 634.2505232845108,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 808.2288258497729,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 887.379851891824,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.8520668127788,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.5722305987382,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 797.2344692851619,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 869.1247893304921,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.3519734489903,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.7524678883676,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.3515369455441,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 801.0417076206649,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.6051195970564,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.3686599154357,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.4226986163708,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 794.2841328092665,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "measured-prices": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.0022900836704,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.9552174966029,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.2109343140315,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 800.0504673403519,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.97894559035507,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.2240166544733,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.9694197067382,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.6122963222863,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.01016860474147,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 640.0832692608378,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 817.2512662810595,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 884.2269914436348,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.7104043697476,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.6818234805448,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.4519626733392,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 871.3791667713392,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.9226963149118,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 633.7833723015253,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 816.4369707828996,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 891.9506536966762,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.46207199869406,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.7867743413317,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.6865724814223,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 865.635197618233,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 357.03236762615666,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.7116359452439,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.2915099254717,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 797.9448746694952,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.6425874924987,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.6005297600364,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 789.4981140441996,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.5205934261813,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 356.01922357491236,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.976694323631,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.7685110062342,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 801.4986519230725,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.2613187916304,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.6761967062737,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 794.976076048426,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 795.080562490925,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.67672827826306,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 632.6198973604828,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.5936174787505,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 884.0863563323007,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.4915866808473,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.2536911028309,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.881160887471,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 863.5395353215404,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.40053191862216,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 633.304800985909,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.0419770200965,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 890.198521413689,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.9898165971873,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.3958309968295,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.5456075523746,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 854.7344795212484,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.1874092587419,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.0248363588287,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.2082939387749,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.786965293864,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.80092292505157,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.4432510106634,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.7560584978334,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.5494222195109,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "marginals-plain-pair": [
+ {
+ "round": 0,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 852.3819711186778,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 806.1499360834738,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 883.6145218099846,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 858.096648332815,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 899.202057482918,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 853.9158417247984,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "suffix",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 850.6813351383993,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "suffix",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 804.1592377201139,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "calibrated-copies": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 356.89736524019355,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.0187324759729,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.741024728307,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 800.8279409915956,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.3997695121801,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.9712879245058,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 791.1426154787024,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 795.5312979171122,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.52706500963103,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 652.9291232633993,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 834.3728340701181,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 896.0390296891412,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.3516920037488,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 607.1620416563453,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 798.5301043013338,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 846.7292730391139,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.29895451504433,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 652.0235878379984,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 835.1149319822576,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 893.3051216132275,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.54066244519555,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 607.1129835703065,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 797.5689915729046,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 851.143583751972,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.0785690913475,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.2397188970659,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.6934232531094,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.3406355080527,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.1190771418354,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5714398365352,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.9064139170582,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.1322137306603,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.8496704697536,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.2522864038602,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.7563792827749,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.9494270904845,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.3042033795154,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.7675067526858,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.0338392986995,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.2412667992925,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 352.7139871670889,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 653.4023148264475,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 833.2547333810215,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 893.8013643122059,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.8086370453564,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 608.1383521142291,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.8971661927403,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 845.9306402571432,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.5316157451344,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 653.3602519209168,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 827.3174545834751,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 883.7418866118552,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 354.71168708573737,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 608.7052764114708,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.1492785220543,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 849.1226837895985,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.5878835834863,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.8822390501088,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.1773804578008,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 800.1935435968933,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.4614062043814,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.77057953921,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 788.3009568824,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 797.1140434478928,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "calibrated-gain": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.2817616928971,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.9503439569837,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 803.8429669050786,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5966217081203,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.32050492397,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 797.9728034474932,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 655.3612739129389,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 827.2921520260961,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 898.5541125939604,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 606.792747118611,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.4692566729715,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 821.9535259414042,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 655.2422011190923,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 828.6847884374228,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 894.6143884256593,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 607.2807456583097,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.5083803054556,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 855.4259251192357,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 648.0985030732918,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 816.4987719920528,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.1220168619425,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.3380293389913,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.8212232749717,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 791.9775841739299,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.8868087939861,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 812.0902189272058,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 797.0857463577217,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.0180434003239,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.494097756085,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 791.2460709694569,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 653.0580187301645,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 835.413492111602,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 895.501733536294,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 607.4044991617268,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.8515252520928,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 832.4465777775944,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 653.4216220485712,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 831.42931875777,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 888.9539789126455,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 608.5829135640643,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 798.6085551885681,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 853.6166076448628,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.6175853573202,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.5462435922141,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.0284264896051,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.2663303179743,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.1055546163396,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 791.8703317129626,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "matched-copies": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 356.99646309324595,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.5492258005319,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.932885862684,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.4129715234225,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.4566124315358,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.514504013522,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.6443901963735,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.6781654196443,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.9166515876299,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 636.5545013305285,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 812.5945113141222,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 884.1927466504886,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.3840436381793,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.1584535260791,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.2157635169305,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 893.7677444400998,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.37209374435827,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 635.9167330704526,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 817.0502363738213,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 884.0799546586949,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 361.10016599986983,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.1828176051994,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 794.5240802161732,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 865.3193294761426,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.909861719153,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.5712374499714,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 814.4099866625494,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 806.3952546542248,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 364.7132316568852,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.8644444174694,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.2592729217582,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 799.9076839089588,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.6164532172002,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.5204569248906,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.3272957694173,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.8247552364493,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.5363752860445,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.1669033437117,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.4741917389516,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.224337544272,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.3822285167143,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 643.1026126925138,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 817.2482925015051,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 887.860611670729,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.2976482608588,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.9026517552672,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.9401589558827,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 863.9163041746773,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 356.5314107668125,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 642.0221936686764,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 818.1466807506825,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 894.9686721601435,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.0136938950127,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 614.8136328635114,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.8489442845871,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 869.8564982576298,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.3171899590034,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.1645619597977,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 815.0506427042201,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.7105753344185,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.6725195491522,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5951679719228,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.3423294154909,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.7533728687552,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "bounded-confidence": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.7680835371117,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.4244961929077,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.3872230675005,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.6378632970965,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.8454649311719,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.0103072043429,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.0616695687706,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.1290787796452,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.23021507526397,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 639.8778292050813,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 818.2102040596474,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 885.2290261607424,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 309.35306237543205,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.7962295598645,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.8960891596205,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 848.1177087601202,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.25533529325133,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 633.9190341655305,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 816.0504998162892,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 888.8641337322866,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 309.14059004467447,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.2354478566609,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 794.6971622280608,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 840.2660430574568,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 352.79436176834565,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.7725475329029,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.6250426792632,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.1671109568664,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.79316424556134,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.8735213883067,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 790.8595630646305,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.2957843037145,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.3220468785289,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.9442405213639,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.3759503855246,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 804.0110891455896,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.20103217979846,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5346952243924,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 789.9944749219242,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 797.1396234203783,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 337.8638929272364,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 633.1106635542818,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 812.8522702261672,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 889.1705482858504,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 309.107753777448,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.7568709687713,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.9285305610407,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 848.7950205273687,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.1499910053361,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 632.756452274153,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.2639894531362,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 892.7542082734848,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 308.4919982383707,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.2449557076353,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.8510800728106,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 841.5034449047317,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.47022188523215,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.9453486741286,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 815.271025412628,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.9898000662508,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.18306525053754,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.3419270640579,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 797.317080241428,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.4595338818068,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "timed-holds": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.9551745170593,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.5637468488261,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 807.984763992905,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 801.3765279189241,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.38041169909104,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5369303550488,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 789.8099675056259,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 794.3285718721822,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 318.8819043039091,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 635.7543080729808,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 812.5681185512982,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 866.5437186969713,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 330.05866983125316,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 599.7307974258288,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.8508341728242,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 852.1405741041499,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 309.5573087694549,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 637.0282173004123,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.9433552924245,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 864.755605766611,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 319.1617018393784,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 606.9957448592583,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 791.4377684287101,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 843.8802420178199,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.5790235851849,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.3876625965851,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.5176633383138,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.4212262894329,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.35965996407475,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.3677654913529,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.1475215096297,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.3545361241212,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 352.9918656139329,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.4623325663915,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.3962240076326,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.0035368390922,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.2320914269661,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.9899757409191,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.6455306086262,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.9715825415828,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 319.64008095683903,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 632.433123684852,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 807.5030357403751,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 888.4345016760495,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 330.9378911041061,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 613.1681927115468,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 798.5116500646026,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 857.913514905577,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 309.67663598459416,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 628.2013924361069,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 800.2961748637547,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 865.7712022424788,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 330.26751182770295,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 607.0239816131244,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 797.2986689404338,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 844.216850754568,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.51257406986366,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.8473415590147,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.826905724031,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.0203905043105,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 355.75225545744615,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.0301509608252,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.9903673990456,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 3,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 795.2775134771482,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "timed-holds-v2": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 356.69230829755,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.353736357899,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.2646631217915,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 800.3407202667829,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 368.6266218893763,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5935735594454,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 798.2022191584463,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 794.2045453854204,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 312.80102959084525,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 630.5094565366052,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.3749866698848,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 883.7831479112893,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 329.9267678631863,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 606.6705659128554,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.2822335082275,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 872.3305412753763,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 309.5502005426481,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 631.7242635070509,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 816.0669308718341,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 889.0877551566958,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 329.0866104337284,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 613.4056960736302,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.3302836453324,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ }
+ ],
+ "timed-holds-v3": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.86114646948226,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.242574970228,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.9454150389603,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 797.3900848713985,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 344.9797439664245,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.8759538569072,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 789.1299924459802,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.2399449618346,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 306.32830748608166,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 630.5045032779684,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.9874900767422,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 892.590961629742,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 317.42741882026826,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 604.5410814558361,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.1636361881042,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 852.6768131157207,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 319.5727336112319,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 637.1253074031247,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.0727058996663,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 885.7462905748039,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 318.3155657011862,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 612.6436896536846,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.8719992041238,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 841.9255394243237,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.3181104124031,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.2442586943916,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.2098888274344,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.6865282121868,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.3090988418972,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.5141289547389,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 788.8790864432178,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.3453621679392,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.0687445295109,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 644.2289378363188,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc"
+ }
+ ]
+ },
+ "incomplete": [
+ "timed-holds-v2",
+ "timed-holds-v3"
+ ],
+ "notes": [
+ "All runs use public code/prose prompts, greedy seed 1234, 256 tokens, context 4096, cache off.",
+ "Generic old/live-price labels mean the frozen published drafter baseline and the private experiment.",
+ "The overhead isolation uses the same frozen binary with suffix, literal draft:false oracle, attached-but-disabled oracle and drafted arms.",
+ "Phase-profile is instrumented and diagnostic; its rates are not release evidence.",
+ "Timed-holds-v2/v3 were explicitly interrupted and are not recorded as completed qualification.",
+ "M4 hold-accounting was interrupted after its controls drifted under unrelated CPU/memory load."
+ ],
+ "m4_interrupted_contended_cells": [
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 255.42985115303972,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 285.8897351673643,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 324.9759499855657,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 230.79741035226908,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 292.03948326150095,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 321.8635097788463,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 250.61604654799328,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 280.35161705555925,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 321.89181286452697,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 233.3064872375181,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 293.4532362980501,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 0,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 322.88252957986236,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 254.92513702804413,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 280.49323867369213,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 316.1699880648688,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 228.74850682582047,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 286.0613003174711,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 295.7224222852996,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 213.9599629051815,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 236.19510069268986,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 268.95459868424456,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 220.88028624348414,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 233.47344218749308,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 1,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 262.46936902073395,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 199.02623810574727,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 178.4980914699512,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 269.9027915810738,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 208.81805728442896,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 217.0602358439617,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "old",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 270.8059638616868,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 211.2455862941568,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 235.13649439640818,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 272.7727044611459,
+ "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 202.21801607515494,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ },
+ {
+ "round": 2,
+ "arm": "live-price",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 230.89681390141425,
+ "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784"
+ }
+ ],
+ "descriptions": {
+ "hold-accounting": "Models outgoing batched heads from the timed shape rather than per-step multiplication.",
+ "overhead-isolation": "Same binary: suffix, literal draft:false oracle, drafter attached but disabled, and active drafting.",
+ "stage512": "128 to 512 staged context rows.",
+ "phase-profile": "Instrumented host-absorb and GPU-hold phase counters, including warmups.",
+ "copy-depth": "Conditional acceptance by copied position, learned separately by source.",
+ "shared-calibration": "Absolute shared-forward timings above 16 rows.",
+ "shared-marginals": "Shared increments anchored to the original 16-row window cost.",
+ "measured-prices": "Shared increments plus outgoing hold-shape overhead accounting.",
+ "marginals-plain-pair": "Same calibrated binary with and without the external drafter.",
+ "calibrated-copies": "Shared increments with the ordinary fixed copy prior restored.",
+ "calibrated-gain": "Shared increments, ordinary copy policy, and gain scaled relative to the acceptance prior.",
+ "matched-copies": "Shared increments with the fixed prior retained for long suffix matches.",
+ "bounded-confidence": "A deeper pending hold falls back entirely when cached confidence is too short.",
+ "timed-holds": "Uniform late-hold budgeting; pending work inferred from selected streams.",
+ "timed-holds-v2": "Uniform budgeting with a drafter-owned pending-cost query; interrupted.",
+ "timed-holds-v3": "Pending-cost metadata and one proposer query; interrupted."
+ },
+ "phase_profile_totals": {
+ "includes_warmups": true,
+ "absorb_calls": 1131,
+ "absorb_rows": 13102,
+ "absorb_wall_ms": 17.757,
+ "explicit_absorb_gpu_ms": 7.58,
+ "hold_calls": 101,
+ "hold_gpu_ms": 434.471,
+ "note": "Hold GPU time also includes staged-context ingestion; these counters are not a pure context-versus-drafting decomposition."
+ }
+}
diff --git a/docs/receipts/lfm2-dspark-review/wide-calibration-candidate.json b/docs/receipts/lfm2-dspark-review/wide-calibration-candidate.json
new file mode 100644
index 0000000000..2f75052526
--- /dev/null
+++ b/docs/receipts/lfm2-dspark-review/wide-calibration-candidate.json
@@ -0,0 +1,16059 @@
+{
+ "schema_version": 1,
+ "date": "2026-10-28",
+ "status": "validated M5 candidate; user accepted the four-stream tradeoff for the larger 16-stream gain; M4 serving qualification is environment-limited",
+ "source": {
+ "candidate": "099455490334f03495cdc67de0bfc72174b552b5",
+ "published": "68c7435b0184a83c7fba458b268bbc5572c0e823",
+ "main": "dd0a9cc5e503c405a71856631bfeabbccd97169f",
+ "sha256": {
+ "zig/src/core/lanes/depth.zig": "06b0c1bfdc1fcb61ad101736e5e7152114ae7f25723b06110910528eafe2728d",
+ "zig/src/core/lanes/drafted_chances_test.zig": "031300891cbbfff3cf9c792b648815eb9475ccb190158335ace665e66dae6023",
+ "zig/src/families/lfm2_moe/backend.zig": "16475175e4ec0392ce46e6c993829664f39c4565888de4388953e0192876ecba",
+ "zig/src/families/lfm2_moe/dspark.zig": "06b5dda907dcb6f44804edfd0c66466bb7422365516613478c777a6e55f46302",
+ "zig/src/families/lfm2_moe/dspark_block.zig": "d2dd077581904759967446ebe2ad195c083ae73c004862de09907ffb4671ac41",
+ "zig/src/core/lanes/windows.zig": "88eedd02202e42e91fcdef4dd046d969038cc391cac282604893a417af3ad13a",
+ "zig/src/core/lanes/stream.zig": "6862f27ff9b86d8f9a6465a5aca8a00403818752e5be947637c806252cff1202",
+ "zig/tests/contract/metal_lfm2_bf16.zig": "06f08a81d48a17371bdaed3b1c036aee085af7d7c0e0eed258b845e51a85c774"
+ }
+ },
+ "device": {
+ "name": "Apple M5 Ultra",
+ "ram_gib": 256,
+ "os": "macOS 27.0 (26A428)",
+ "metal_compiler": "32023.921.6",
+ "zig": "0.17.0",
+ "build_options": [
+ "-Dcpu=apple_m1",
+ "-Dversion=1.1.8"
+ ]
+ },
+ "checkpoint": {
+ "target": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit",
+ "revision": "667408d494bde191c2cb9dd4f4549827816b9275",
+ "drafter": "LiquidAI/LFM2.5-24B-A2B-DSpark",
+ "drafter_bits": 0,
+ "sha256": {
+ "drafter": {
+ "model.safetensors": "7d06a78556b0b1f386134508f09b00a3a7e0e5e0ec2cc73119229f4ef7c9b299"
+ },
+ "tokenizer.json": "695be7802a0e4b8a81048f0ff5ebb7fc811a0ba5a6be63dbb24deb5a81096f41"
+ }
+ },
+ "prompts": [
+ {
+ "name": "code",
+ "prompt": "Write a short Python function that computes the Fibonacci sequence and explain it.",
+ "count": 14,
+ "tokens": [
+ 22497,
+ 267,
+ 2789,
+ 21516,
+ 1446,
+ 395,
+ 91610,
+ 278,
+ 91230,
+ 8055,
+ 309,
+ 6986,
+ 435,
+ 22
+ ]
+ },
+ {
+ "name": "prose",
+ "prompt": "Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.",
+ "count": 18,
+ "tokens": [
+ 89509,
+ 1030,
+ 9706,
+ 37207,
+ 5081,
+ 267,
+ 54340,
+ 296,
+ 15175,
+ 3478,
+ 20,
+ 1256,
+ 3118,
+ 267,
+ 1710,
+ 20819,
+ 2122,
+ 22
+ ]
+ }
+ ],
+ "binaries_sha256": {
+ "main": "3c67809dbda6c2735279fcb0bf084ec21adef406f1f27c6bbbb86bac213e30a5",
+ "published": "56221194e4dd9f86a030a916fd7971998013089b9023ce72e8cce11c633f81db",
+ "candidate-plain": "383a32879ec4d4e2c6f8f65bc21de76993c730032a937ce66b996485efa55b7a",
+ "candidate-drafted": "383a32879ec4d4e2c6f8f65bc21de76993c730032a937ce66b996485efa55b7a"
+ },
+ "protocol": {
+ "context": 4096,
+ "parallel_limit": 16,
+ "prompt_cache_gib": 0,
+ "max_tokens": 256,
+ "warmup_tokens": 32,
+ "temperature": 0,
+ "seed": 1234,
+ "ignore_eos": true,
+ "repetitions": 4,
+ "aggregate_decode_tps": "streams * 255 / (latest last SSE token - earliest first SSE token)",
+ "main_and_candidate_plain": "no external drafter; suffix copying remains enabled",
+ "oracle_controls": "literal draft:false is used separately for correctness",
+ "arm_order_by_round": [
+ [
+ "main",
+ "published",
+ "candidate-plain",
+ "candidate-drafted"
+ ],
+ [
+ "published",
+ "candidate-plain",
+ "candidate-drafted",
+ "main"
+ ],
+ [
+ "candidate-plain",
+ "candidate-drafted",
+ "main",
+ "published"
+ ],
+ [
+ "candidate-drafted",
+ "main",
+ "published",
+ "candidate-plain"
+ ]
+ ]
+ },
+ "cells": [
+ {
+ "round": 0,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 289.3949382948194,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019017125000004853,
+ "last_s": 0.900165916000006,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 652.4907022857611,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.04169999955684e-05,
+ "first_s": 0.05019108399999084,
+ "last_s": 1.576910374999997,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00022637499999689226,
+ "first_s": 0.033862124999998855,
+ "last_s": 1.5768938340000034,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00016599999999300508,
+ "first_s": 0.06655641700000103,
+ "last_s": 1.5807140000000004,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.017473334000001728,
+ "last_s": 1.5768808339999936,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 830.9188714751687,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.087499999793636e-05,
+ "first_s": 0.03411058300000036,
+ "last_s": 2.4651412080000057,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0004351249999956508,
+ "first_s": 0.09942162500000506,
+ "last_s": 2.4656670410000032,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0005259160000008478,
+ "first_s": 0.08311704099999417,
+ "last_s": 2.465258457999994,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00016016600000057224,
+ "first_s": 0.1320782079999958,
+ "last_s": 2.4728565409999987,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.00032724999999800275,
+ "first_s": 0.06678016600000092,
+ "last_s": 2.4654044159999984,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0002673329999964835,
+ "first_s": 0.050419457999993256,
+ "last_s": 2.4652741249999934,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0002102499999949714,
+ "first_s": 0.017743207999998845,
+ "last_s": 2.4651232909999976,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.11576108299999532,
+ "last_s": 2.465467583000006,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 830.6686179686005,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 8.850000000393266e-05,
+ "first_s": 0.19850612500000864,
+ "last_s": 4.930106042000006,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0007719590000050403,
+ "first_s": 0.10046120900000233,
+ "last_s": 4.929726459000008,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.000659624999997277,
+ "first_s": 0.11679204200000015,
+ "last_s": 4.929789749999998,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0005466249999983575,
+ "first_s": 0.08411750000000495,
+ "last_s": 4.9300628339999975,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0001528749999977208,
+ "first_s": 0.13311520899999607,
+ "last_s": 4.929924999999997,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00024087500000291584,
+ "first_s": 0.21486808400000257,
+ "last_s": 4.930217917000007,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0009855420000093318,
+ "first_s": 0.23116025000000207,
+ "last_s": 4.929859833999998,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00028745900000615165,
+ "first_s": 0.03504391699999587,
+ "last_s": 4.9292779999999965,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0003365419999994401,
+ "first_s": 0.16581237500000157,
+ "last_s": 4.930373000000003,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0004827500000033069,
+ "first_s": 0.14947666700000184,
+ "last_s": 4.9299721670000025,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0007235839999992777,
+ "first_s": 0.2638151250000078,
+ "last_s": 4.930016875000007,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0003879169999976284,
+ "first_s": 0.06777829199999985,
+ "last_s": 4.929429917000007,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0009317089999996142,
+ "first_s": 0.18218070900000782,
+ "last_s": 4.930336875000009,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0008437919999977339,
+ "first_s": 0.051403542000002744,
+ "last_s": 4.929457584000005,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00043537500000923046,
+ "first_s": 0.01866704200000413,
+ "last_s": 4.929269458999997,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.24748458400000573,
+ "last_s": 4.930299667,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 262.6130764018713,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020124791999990066,
+ "last_s": 0.9911350839999926,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 622.8271495637517,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.204199999146567e-05,
+ "first_s": 0.03935583399999132,
+ "last_s": 1.6581554589999996,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00023499999998932708,
+ "first_s": 0.058240541999992956,
+ "last_s": 1.658177166999991,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00016199999998889325,
+ "first_s": 0.0770006249999966,
+ "last_s": 1.6581916250000006,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020498083999996197,
+ "last_s": 1.6581663749999933,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 804.5637829924607,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.06250000127784e-05,
+ "first_s": 0.05845654100001241,
+ "last_s": 2.552360875000005,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003829580000029864,
+ "first_s": 0.15243112500000677,
+ "last_s": 2.556268958000004,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00026512500001274475,
+ "first_s": 0.020733500000005733,
+ "last_s": 2.552292458000011,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00021220800000776308,
+ "first_s": 0.09606691600001227,
+ "last_s": 2.5528507500000046,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00016045800001052157,
+ "first_s": 0.03959083300000543,
+ "last_s": 2.5523156250000056,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00046087500000169257,
+ "first_s": 0.13367604100000108,
+ "last_s": 2.5526933330000077,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003414160000119182,
+ "first_s": 0.11484554100000821,
+ "last_s": 2.5525915830000088,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.0772501659999989,
+ "last_s": 2.5524381660000017,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 799.7552905625633,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.183400000267739e-05,
+ "first_s": 0.2845150840000059,
+ "last_s": 5.1223552090000055,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005195000000099981,
+ "first_s": 0.021076792000002342,
+ "last_s": 5.12148387500001,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00015887499999678312,
+ "first_s": 0.26571837500000584,
+ "last_s": 5.122637292000007,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00020879200000933906,
+ "first_s": 0.15295966700000463,
+ "last_s": 5.122318042000003,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00040704200000618584,
+ "first_s": 0.03995979200000477,
+ "last_s": 5.121496000000008,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00045674999999789634,
+ "first_s": 0.096516500000007,
+ "last_s": 5.122189958999996,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007996249999990823,
+ "first_s": 0.07765324999999734,
+ "last_s": 5.121597500000007,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003106670000079248,
+ "first_s": 0.058872542000003136,
+ "last_s": 5.121506625000009,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00026083400000231904,
+ "first_s": 0.24694041700000469,
+ "last_s": 5.122128750000002,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000634458999996923,
+ "first_s": 0.1341646670000074,
+ "last_s": 5.121775334000006,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005756249999961938,
+ "first_s": 0.30331350000000157,
+ "last_s": 5.122497584000001,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007446250000100463,
+ "first_s": 0.22816958399999976,
+ "last_s": 5.122397583999998,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00035950000000184446,
+ "first_s": 0.11529016699999772,
+ "last_s": 5.121981042000002,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0008717090000089911,
+ "first_s": 0.1717496250000039,
+ "last_s": 5.12244920900001,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006986249999982874,
+ "first_s": 0.19057083399999897,
+ "last_s": 5.122595333999996,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.20936879200000647,
+ "last_s": 5.121805292000005,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 352.5059309330252,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01938350000000355,
+ "last_s": 0.7427755840000003,
+ "chunks": 90,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 89,
+ "drafted": 305,
+ "accepted": 166,
+ "acceptance_rate": 0.5442622950819672,
+ "tokens_per_round": 2.8764044943820224
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.038823928742,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.658300000126019e-05,
+ "first_s": 0.052342875000007894,
+ "last_s": 1.5994826670000037,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00023475000000416912,
+ "first_s": 0.018182500000008872,
+ "last_s": 1.5955713330000094,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00016962499999806369,
+ "first_s": 0.035265708000011387,
+ "last_s": 1.595589583000006,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 41,
+ "accepted": 32,
+ "acceptance_rate": 0.7804878048780488,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.0693992080000072,
+ "last_s": 1.5894960830000002,
+ "chunks": 223,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 222,
+ "drafted": 41,
+ "accepted": 33,
+ "acceptance_rate": 0.8048780487804879,
+ "tokens_per_round": 1.1531531531531531
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 808.7487720963637,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.295899999983703e-05,
+ "first_s": 0.05481666699999721,
+ "last_s": 2.5259354169999995,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.00016308399999331868,
+ "first_s": 0.12275454199999558,
+ "last_s": 2.525838209,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 29,
+ "accepted": 22,
+ "acceptance_rate": 0.7586206896551724,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.00039104199998973854,
+ "first_s": 0.10571129199999518,
+ "last_s": 2.525754499999991,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.00021399999999971442,
+ "first_s": 0.037824791999995,
+ "last_s": 2.5255334169999912,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0004700839999998152,
+ "first_s": 0.13968074999999658,
+ "last_s": 2.540863791999996,
+ "chunks": 238,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 237,
+ "drafted": 24,
+ "accepted": 18,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.080168776371308
+ }
+ },
+ {
+ "sent_s": 0.000267624999992222,
+ "first_s": 0.018448833999997305,
+ "last_s": 2.525459541999993,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.00033166699999753746,
+ "first_s": 0.07177470899999605,
+ "last_s": 2.5255709999999993,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.0887991249999942,
+ "last_s": 2.525884167000001,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.9815660837332,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.779100000173457e-05,
+ "first_s": 0.06963183300000253,
+ "last_s": 5.124508708000008,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0010130410000073198,
+ "first_s": 0.12072145800000555,
+ "last_s": 5.124827791000001,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0006217079999970565,
+ "first_s": 0.18884545800000296,
+ "last_s": 5.1249649579999925,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0005764579999976149,
+ "first_s": 0.273867249999995,
+ "last_s": 5.124401290999998,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00017737500000691853,
+ "first_s": 0.17184645800000453,
+ "last_s": 5.124922749999996,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002988330000022188,
+ "first_s": 0.2058704580000068,
+ "last_s": 5.125055875000015,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002442079999980251,
+ "first_s": 0.01855508299999542,
+ "last_s": 5.123774832999999,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0008294999999947095,
+ "first_s": 0.08668137499999773,
+ "last_s": 5.124786916000005,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.000514166000002092,
+ "first_s": 0.137795416000003,
+ "last_s": 5.124281749999994,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0006679159999976036,
+ "first_s": 0.052570290999994995,
+ "last_s": 5.1237929160000135,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0003599580000042124,
+ "first_s": 0.23988308299999517,
+ "last_s": 5.124696707999988,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0007899160000022221,
+ "first_s": 0.1547861250000011,
+ "last_s": 5.124875666000008,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0004132079999976668,
+ "first_s": 0.10369641600000534,
+ "last_s": 5.1250158750000026,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00046420799999680185,
+ "first_s": 0.03555520800000522,
+ "last_s": 5.123854375000008,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.001606541000001016,
+ "first_s": 0.2568893330000037,
+ "last_s": 5.124641999999994,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.22285541600000158,
+ "last_s": 5.12459233300001,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.46165642120616,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.02022737499999039,
+ "last_s": 0.7355918339999903,
+ "chunks": 92,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.1953840111927,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.26660000004631e-05,
+ "first_s": 0.039795416000004025,
+ "last_s": 1.6702774999999974,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0002373750000117525,
+ "first_s": 0.07868741600000817,
+ "last_s": 1.6703163749999987,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00016404100000499966,
+ "first_s": 0.020352583000004643,
+ "last_s": 1.6702853330000096,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.05923737500000925,
+ "last_s": 1.6702927500000158,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 788.127553808539,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.304200000315177e-05,
+ "first_s": 0.15684191700000838,
+ "last_s": 2.599768291999993,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 10,
+ "accepted": 4,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0004981250000071213,
+ "first_s": 0.1374700419999897,
+ "last_s": 2.5997595839999974,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 11,
+ "accepted": 4,
+ "acceptance_rate": 0.36363636363636365,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00033579199998712284,
+ "first_s": 0.1180892090000043,
+ "last_s": 2.609401208999998,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 6,
+ "accepted": 2,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.00026162500000737055,
+ "first_s": 0.020987708999996357,
+ "last_s": 2.593058667000008,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 15,
+ "accepted": 5,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00015808399999173162,
+ "first_s": 0.09872774999999478,
+ "last_s": 2.6093892499999924,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 6,
+ "accepted": 2,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.00038604200000236233,
+ "first_s": 0.07931337499999813,
+ "last_s": 2.5932052919999933,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 8,
+ "accepted": 5,
+ "acceptance_rate": 0.625,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00020895899999118228,
+ "first_s": 0.059855709000004254,
+ "last_s": 2.593099125000009,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 8,
+ "accepted": 5,
+ "acceptance_rate": 0.625,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.04044404199999008,
+ "last_s": 2.5930863339999917,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 9,
+ "accepted": 5,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.024
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.5756383104019,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.562500000015461e-05,
+ "first_s": 0.15681554200000392,
+ "last_s": 5.168663334000001,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0007145419999972091,
+ "first_s": 0.2733495420000054,
+ "last_s": 5.168337708999985,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0003699589999826003,
+ "first_s": 0.23450595899998916,
+ "last_s": 5.168161709000003,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00021504200000777018,
+ "first_s": 0.07925975000000562,
+ "last_s": 5.167919041999994,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00016416700000831952,
+ "first_s": 0.13745066699999597,
+ "last_s": 5.168629041999992,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00041904200000431047,
+ "first_s": 0.11803787499999885,
+ "last_s": 5.168748625000006,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0005311669999912283,
+ "first_s": 0.05986920899999859,
+ "last_s": 5.1685104999999965,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0010202499999820702,
+ "first_s": 0.19564241699998774,
+ "last_s": 5.168433416999989,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0002687089999824366,
+ "first_s": 0.02104024999999865,
+ "last_s": 5.167743124999987,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0006473749999997835,
+ "first_s": 0.21508354200000213,
+ "last_s": 5.168813916999994,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0003192089999970449,
+ "first_s": 0.04043733399998928,
+ "last_s": 5.167883125000003,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.000785250000006954,
+ "first_s": 0.2927850000000092,
+ "last_s": 5.168283833999993,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.000600833999982342,
+ "first_s": 0.2539376670000024,
+ "last_s": 5.168594999999982,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0009392919999982041,
+ "first_s": 0.31220487499999194,
+ "last_s": 5.168086458999994,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0004664999999874908,
+ "first_s": 0.09865124999998898,
+ "last_s": 5.168122667000006,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.17620324999998616,
+ "last_s": 5.168553541999984,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 289.664898526758,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01763933300000531,
+ "last_s": 0.8979669160000014,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 661.8283265634324,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00011133300000665258,
+ "first_s": 0.03458058300000744,
+ "last_s": 1.5593261660000053,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 63,
+ "accepted": 37,
+ "acceptance_rate": 0.5873015873015873,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.0002732499999922311,
+ "first_s": 0.0673262500000078,
+ "last_s": 1.5593379999999968,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.00019895799999858355,
+ "first_s": 0.050959957999992866,
+ "last_s": 1.5593013330000076,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.018152832999987822,
+ "last_s": 1.5593156660000034,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 63,
+ "accepted": 37,
+ "acceptance_rate": 0.5873015873015873,
+ "tokens_per_round": 1.1743119266055047
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 852.2134767889547,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.850000000710679e-05,
+ "first_s": 0.10013691599999675,
+ "last_s": 2.4077089580000006,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0005052500000033433,
+ "first_s": 0.13276054100001033,
+ "last_s": 2.4121705409999947,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0004542910000111533,
+ "first_s": 0.116439583000016,
+ "last_s": 2.4121618750000096,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0003588749999892116,
+ "first_s": 0.05117420799999195,
+ "last_s": 2.4075726250000002,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0001736250000021755,
+ "first_s": 0.018404124999989335,
+ "last_s": 2.4072412080000163,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0002284159999987878,
+ "first_s": 0.06752266600000212,
+ "last_s": 2.407417874999993,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0002847080000094593,
+ "first_s": 0.03478341600001045,
+ "last_s": 2.407264583,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.08383995799999866,
+ "last_s": 2.407447458000007,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 849.1748999161405,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00010283399998911591,
+ "first_s": 0.11666416700001037,
+ "last_s": 4.814435916999997,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.000866624999986243,
+ "first_s": 0.16564487499999814,
+ "last_s": 4.814162417000006,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00017866700000013225,
+ "first_s": 0.08403904200000056,
+ "last_s": 4.813875999999993,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00023091700001032223,
+ "first_s": 0.0677802090000057,
+ "last_s": 4.813584166999988,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00034616700000356104,
+ "first_s": 0.10040054199998849,
+ "last_s": 4.814117374999995,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0008200419999866426,
+ "first_s": 0.19820616699999505,
+ "last_s": 4.814219667000003,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0005775420000020404,
+ "first_s": 0.13295387499999833,
+ "last_s": 4.814350917000013,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00028962499999352076,
+ "first_s": 0.0350679590000027,
+ "last_s": 4.813511459000011,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0006246670000109589,
+ "first_s": 0.21453454199999555,
+ "last_s": 4.813862416999996,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0004491669999993064,
+ "first_s": 0.26347254199998815,
+ "last_s": 4.8232971250000105,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 49,
+ "accepted": 19,
+ "acceptance_rate": 0.3877551020408163,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.0007704589999946165,
+ "first_s": 0.05141445900000008,
+ "last_s": 4.813496708999992,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0006760420000091472,
+ "first_s": 0.23084758400000283,
+ "last_s": 4.814389709000011,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.000498625000005859,
+ "first_s": 0.1819189999999935,
+ "last_s": 4.81455895900001,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0003985840000098051,
+ "first_s": 0.1493125419999899,
+ "last_s": 4.814299833999996,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0007239169999877504,
+ "first_s": 0.24718016700001044,
+ "last_s": 4.823283500000002,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 49,
+ "accepted": 19,
+ "acceptance_rate": 0.3877551020408163,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01863320900000076,
+ "last_s": 4.813484875,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 261.38390992402503,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.02007799999998383,
+ "last_s": 0.9956545000000006,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 609.9669343383413,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.233299999777955e-05,
+ "first_s": 0.07724037500000236,
+ "last_s": 1.6928297499999871,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.00025262500000167165,
+ "first_s": 0.03951279099999283,
+ "last_s": 1.692698665999984,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 35,
+ "accepted": 8,
+ "acceptance_rate": 0.22857142857142856,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.00016170799997894392,
+ "first_s": 0.02060795799999937,
+ "last_s": 1.6926645000000065,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 35,
+ "accepted": 8,
+ "acceptance_rate": 0.22857142857142856,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.058399415999986104,
+ "last_s": 1.692706749999985,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 809.672115751542,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.00010250000002542947,
+ "first_s": 0.05841479100001834,
+ "last_s": 2.539832791000009,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0004066660000034972,
+ "first_s": 0.13364954100001114,
+ "last_s": 2.540146375000006,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00018041600000628932,
+ "first_s": 0.03954866600000173,
+ "last_s": 2.5398536660000275,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00046229100001937695,
+ "first_s": 0.09605575000000499,
+ "last_s": 2.5398657500000184,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0005358330000149181,
+ "first_s": 0.11484220800002731,
+ "last_s": 2.5402444160000073,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0002577500000029431,
+ "first_s": 0.07728983300000891,
+ "last_s": 2.5399477079999997,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00034770800002092983,
+ "first_s": 0.1524370410000131,
+ "last_s": 2.5402004160000047,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.02070600000001832,
+ "last_s": 2.539821083000021,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 801.9628172039064,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.045800001672433e-05,
+ "first_s": 0.021071375000019543,
+ "last_s": 5.103436583000018,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007646250000163946,
+ "first_s": 0.2842919160000008,
+ "last_s": 5.108578083000026,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 35,
+ "accepted": 3,
+ "acceptance_rate": 0.08571428571428572,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002085410000063348,
+ "first_s": 0.1715162080000141,
+ "last_s": 5.103870625000013,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005340000000160217,
+ "first_s": 0.11514520800000128,
+ "last_s": 5.104300083000027,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00015829100001951701,
+ "first_s": 0.20912250000000654,
+ "last_s": 5.1037983330000145,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00025845800001889074,
+ "first_s": 0.05869633300000032,
+ "last_s": 5.103474958000021,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00040904100001171173,
+ "first_s": 0.24668070800001374,
+ "last_s": 5.104044791000007,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006332910000139691,
+ "first_s": 0.09634137500000861,
+ "last_s": 5.104135041000006,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005792910000081974,
+ "first_s": 0.1527348330000109,
+ "last_s": 5.104255750000021,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00035791600001289225,
+ "first_s": 0.2655055830000208,
+ "last_s": 5.10409525,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000925416000001178,
+ "first_s": 0.22789587500000152,
+ "last_s": 5.104215083000014,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003089160000229185,
+ "first_s": 0.03990633300000468,
+ "last_s": 5.103451416000013,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007230410000147458,
+ "first_s": 0.13391554100002168,
+ "last_s": 5.103779375000016,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006800410000096235,
+ "first_s": 0.3030480830000215,
+ "last_s": 5.108589041000016,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 35,
+ "accepted": 3,
+ "acceptance_rate": 0.08571428571428572,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00045841600001494953,
+ "first_s": 0.07749095800002692,
+ "last_s": 5.103549541000007,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.19033883300002685,
+ "last_s": 5.104173500000002,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 354.03697603829806,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019182792000009385,
+ "last_s": 0.739446541999996,
+ "chunks": 90,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 89,
+ "drafted": 305,
+ "accepted": 166,
+ "acceptance_rate": 0.5442622950819672,
+ "tokens_per_round": 2.8764044943820224
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 632.6567488487849,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00010625000001596163,
+ "first_s": 0.06942499999999541,
+ "last_s": 1.630453541999998,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 54,
+ "accepted": 33,
+ "acceptance_rate": 0.6111111111111112,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0002620000000206346,
+ "first_s": 0.03533179199999381,
+ "last_s": 1.605780167000006,
+ "chunks": 221,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 220,
+ "drafted": 53,
+ "accepted": 35,
+ "acceptance_rate": 0.660377358490566,
+ "tokens_per_round": 1.1636363636363636
+ }
+ },
+ {
+ "sent_s": 0.00018633300001624775,
+ "first_s": 0.05237804200001506,
+ "last_s": 1.6304402080000102,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 55,
+ "accepted": 33,
+ "acceptance_rate": 0.6,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.018204875000009224,
+ "last_s": 1.6057571249999967,
+ "chunks": 221,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 220,
+ "drafted": 54,
+ "accepted": 35,
+ "acceptance_rate": 0.6481481481481481,
+ "tokens_per_round": 1.1636363636363636
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 816.3843674754631,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.166699999241246e-05,
+ "first_s": 0.08664674999999988,
+ "last_s": 2.4797165829999983,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 41,
+ "accepted": 30,
+ "acceptance_rate": 0.7317073170731707,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.000474624999981188,
+ "first_s": 0.12073754200000053,
+ "last_s": 2.461433707999987,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 46,
+ "accepted": 32,
+ "acceptance_rate": 0.6956521739130435,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0001684169999975893,
+ "first_s": 0.018413541999990457,
+ "last_s": 2.4714994579999825,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 42,
+ "accepted": 31,
+ "acceptance_rate": 0.7380952380952381,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00022629199997936666,
+ "first_s": 0.06960504199997786,
+ "last_s": 2.4797044169999936,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 44,
+ "accepted": 30,
+ "acceptance_rate": 0.6818181818181818,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0003487500000005639,
+ "first_s": 0.05257195799998726,
+ "last_s": 2.4715415829999756,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 41,
+ "accepted": 31,
+ "acceptance_rate": 0.7560975609756098,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0004019999999798074,
+ "first_s": 0.10369166699999255,
+ "last_s": 2.479870874999989,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 39,
+ "accepted": 30,
+ "acceptance_rate": 0.7692307692307693,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0002774999999815009,
+ "first_s": 0.1377821250000011,
+ "last_s": 2.5172364999999957,
+ "chunks": 230,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 229,
+ "drafted": 42,
+ "accepted": 26,
+ "acceptance_rate": 0.6190476190476191,
+ "tokens_per_round": 1.1179039301310043
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.035519082999996954,
+ "last_s": 2.4715244169999835,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 41,
+ "accepted": 31,
+ "acceptance_rate": 0.7560975609756098,
+ "tokens_per_round": 1.1428571428571428
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 886.0534135309714,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.408400001120754e-05,
+ "first_s": 0.03568766700001902,
+ "last_s": 4.383103042000016,
+ "chunks": 209,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 208,
+ "drafted": 68,
+ "accepted": 47,
+ "acceptance_rate": 0.6911764705882353,
+ "tokens_per_round": 1.2307692307692308
+ }
+ },
+ {
+ "sent_s": 0.0008179170000062186,
+ "first_s": 0.18918575000000715,
+ "last_s": 4.623324875000009,
+ "chunks": 229,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 228,
+ "drafted": 39,
+ "accepted": 27,
+ "acceptance_rate": 0.6923076923076923,
+ "tokens_per_round": 1.1228070175438596
+ }
+ },
+ {
+ "sent_s": 0.00016258400000879192,
+ "first_s": 0.08681654200000821,
+ "last_s": 4.619348416999998,
+ "chunks": 228,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 227,
+ "drafted": 46,
+ "accepted": 28,
+ "acceptance_rate": 0.6086956521739131,
+ "tokens_per_round": 1.1277533039647578
+ }
+ },
+ {
+ "sent_s": 0.0005443750000040382,
+ "first_s": 0.22330387499999915,
+ "last_s": 4.593885999999998,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 45,
+ "accepted": 32,
+ "acceptance_rate": 0.7111111111111111,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0002161670000191407,
+ "first_s": 0.2574249170000087,
+ "last_s": 4.025517334,
+ "chunks": 191,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 190,
+ "drafted": 93,
+ "accepted": 65,
+ "acceptance_rate": 0.6989247311827957,
+ "tokens_per_round": 1.3473684210526315
+ }
+ },
+ {
+ "sent_s": 0.0002708339999912823,
+ "first_s": 0.20626833400001487,
+ "last_s": 4.607208041999996,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 39,
+ "accepted": 30,
+ "acceptance_rate": 0.7692307692307693,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0003712920000111808,
+ "first_s": 0.0697367920000147,
+ "last_s": 4.578519083999993,
+ "chunks": 222,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 221,
+ "drafted": 50,
+ "accepted": 34,
+ "acceptance_rate": 0.68,
+ "tokens_per_round": 1.158371040723982
+ }
+ },
+ {
+ "sent_s": 0.0009464999999977408,
+ "first_s": 0.137989417,
+ "last_s": 4.4582448339999985,
+ "chunks": 214,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 213,
+ "drafted": 50,
+ "accepted": 42,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.2018779342723005
+ }
+ },
+ {
+ "sent_s": 0.0005899170000134291,
+ "first_s": 0.27448724999999286,
+ "last_s": 4.442635249999995,
+ "chunks": 213,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 212,
+ "drafted": 62,
+ "accepted": 43,
+ "acceptance_rate": 0.6935483870967742,
+ "tokens_per_round": 1.2075471698113207
+ }
+ },
+ {
+ "sent_s": 0.00042354199999294906,
+ "first_s": 0.2403498749999926,
+ "last_s": 4.619380709000012,
+ "chunks": 228,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 227,
+ "drafted": 46,
+ "accepted": 28,
+ "acceptance_rate": 0.6086956521739131,
+ "tokens_per_round": 1.1277533039647578
+ }
+ },
+ {
+ "sent_s": 0.0003216670000085742,
+ "first_s": 0.052726875000018936,
+ "last_s": 4.49220895900001,
+ "chunks": 216,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 215,
+ "drafted": 60,
+ "accepted": 40,
+ "acceptance_rate": 0.6666666666666666,
+ "tokens_per_round": 1.1906976744186046
+ }
+ },
+ {
+ "sent_s": 0.0006358340000076623,
+ "first_s": 0.12092820899999879,
+ "last_s": 4.244078042000012,
+ "chunks": 202,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 201,
+ "drafted": 66,
+ "accepted": 54,
+ "acceptance_rate": 0.8181818181818182,
+ "tokens_per_round": 1.2736318407960199
+ }
+ },
+ {
+ "sent_s": 0.0007048340000039843,
+ "first_s": 0.10387600000001385,
+ "last_s": 4.427200499999998,
+ "chunks": 212,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 211,
+ "drafted": 62,
+ "accepted": 44,
+ "acceptance_rate": 0.7096774193548387,
+ "tokens_per_round": 1.2132701421800949
+ }
+ },
+ {
+ "sent_s": 0.0007632920000162358,
+ "first_s": 0.15505937500000755,
+ "last_s": 4.367775167000019,
+ "chunks": 208,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 207,
+ "drafted": 60,
+ "accepted": 48,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.2367149758454106
+ }
+ },
+ {
+ "sent_s": 0.0004777919999980895,
+ "first_s": 0.018636334000007082,
+ "last_s": 4.222254334000013,
+ "chunks": 201,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 200,
+ "drafted": 87,
+ "accepted": 55,
+ "acceptance_rate": 0.632183908045977,
+ "tokens_per_round": 1.28
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.17210966700000085,
+ "last_s": 4.458342250000015,
+ "chunks": 214,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 213,
+ "drafted": 51,
+ "accepted": 42,
+ "acceptance_rate": 0.8235294117647058,
+ "tokens_per_round": 1.2018779342723005
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.0286814706803,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020285457999989376,
+ "last_s": 0.7345137910000119,
+ "chunks": 92,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.2343022307355,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.7167000006948e-05,
+ "first_s": 0.020550209000020914,
+ "last_s": 1.678386209000024,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00024150000001554872,
+ "first_s": 0.0787438750000149,
+ "last_s": 1.6784551670000099,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00017075000002364504,
+ "first_s": 0.05935804200001371,
+ "last_s": 1.678423500000008,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.040020125000012285,
+ "last_s": 1.6783760000000143,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 16,
+ "accepted": 6,
+ "acceptance_rate": 0.375,
+ "tokens_per_round": 1.0281124497991967
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 794.7774420026692,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.837499999321153e-05,
+ "first_s": 0.04286704099999383,
+ "last_s": 2.573487790999991,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00035412500000120417,
+ "first_s": 0.14019941600000152,
+ "last_s": 2.5737503329999925,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 16,
+ "accepted": 5,
+ "acceptance_rate": 0.3125,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00017374999998764906,
+ "first_s": 0.021025666000014098,
+ "last_s": 2.5734014580000064,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.000523958000002267,
+ "first_s": 0.15965133299999934,
+ "last_s": 2.5739015830000085,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 15,
+ "accepted": 5,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00022058299998661823,
+ "first_s": 0.06268308299999603,
+ "last_s": 2.573841625,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0004306659999997464,
+ "first_s": 0.08208716600000798,
+ "last_s": 2.5877701249999916,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 5,
+ "accepted": 2,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.00028129099999318896,
+ "first_s": 0.10145412500000361,
+ "last_s": 2.5877819579999937,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 5,
+ "accepted": 2,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.12079104099998972,
+ "last_s": 2.573796291000008,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 9,
+ "accepted": 5,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.024
+ }
+ }
+ ]
+ },
+ {
+ "round": 0,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 868.6711590769241,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 8.066699999176308e-05,
+ "first_s": 0.1375496669999734,
+ "last_s": 4.6933760829999756,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 19,
+ "accepted": 8,
+ "acceptance_rate": 0.42105263157894735,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.00014508299997828544,
+ "first_s": 0.11816745799998785,
+ "last_s": 4.600768082999991,
+ "chunks": 240,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 30,
+ "accepted": 16,
+ "acceptance_rate": 0.5333333333333333,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00090862499999389,
+ "first_s": 0.3120279579999874,
+ "last_s": 4.717991749999982,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 16,
+ "accepted": 5,
+ "acceptance_rate": 0.3125,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0005382919999874503,
+ "first_s": 0.21509041699999898,
+ "last_s": 4.717809416999984,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00020049999997695522,
+ "first_s": 0.25389033299998687,
+ "last_s": 4.693535666999992,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 14,
+ "accepted": 8,
+ "acceptance_rate": 0.5714285714285714,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.0002541249999978845,
+ "first_s": 0.02116262499998811,
+ "last_s": 4.710796749999986,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 22,
+ "accepted": 6,
+ "acceptance_rate": 0.2727272727272727,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00035341699998525655,
+ "first_s": 0.17631412499997623,
+ "last_s": 4.71082741699999,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 20,
+ "accepted": 6,
+ "acceptance_rate": 0.3,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0007031669999832957,
+ "first_s": 0.07942274999999199,
+ "last_s": 4.11868149999998,
+ "chunks": 206,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 205,
+ "drafted": 75,
+ "accepted": 50,
+ "acceptance_rate": 0.6666666666666666,
+ "tokens_per_round": 1.248780487804878
+ }
+ },
+ {
+ "sent_s": 0.0007799169999884725,
+ "first_s": 0.2344351249999761,
+ "last_s": 4.717902916999975,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 11,
+ "accepted": 5,
+ "acceptance_rate": 0.45454545454545453,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0005970829999739635,
+ "first_s": 0.195703457999997,
+ "last_s": 4.717783082999972,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00030537499998217754,
+ "first_s": 0.27325720799998976,
+ "last_s": 4.310620916999994,
+ "chunks": 219,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 45,
+ "accepted": 37,
+ "acceptance_rate": 0.8222222222222222,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.0004062919999796577,
+ "first_s": 0.09882720799998879,
+ "last_s": 4.454720457999997,
+ "chunks": 229,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 228,
+ "drafted": 43,
+ "accepted": 27,
+ "acceptance_rate": 0.627906976744186,
+ "tokens_per_round": 1.1228070175438596
+ }
+ },
+ {
+ "sent_s": 0.0008489999999881093,
+ "first_s": 0.29266841699998736,
+ "last_s": 3.9745678329999805,
+ "chunks": 197,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 196,
+ "drafted": 76,
+ "accepted": 59,
+ "acceptance_rate": 0.7763157894736842,
+ "tokens_per_round": 1.3061224489795917
+ }
+ },
+ {
+ "sent_s": 0.0006461249999745178,
+ "first_s": 0.040601624999993646,
+ "last_s": 4.600757082999991,
+ "chunks": 240,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 34,
+ "accepted": 16,
+ "acceptance_rate": 0.47058823529411764,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00045437499997547093,
+ "first_s": 0.15691949999998656,
+ "last_s": 4.682362041999994,
+ "chunks": 247,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 246,
+ "drafted": 26,
+ "accepted": 9,
+ "acceptance_rate": 0.34615384615384615,
+ "tokens_per_round": 1.0406504065040652
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.060019041999993306,
+ "last_s": 4.702711332999996,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 15,
+ "accepted": 7,
+ "acceptance_rate": 0.4666666666666667,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 353.875518041669,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019534999999990532,
+ "last_s": 0.7401273749999859,
+ "chunks": 90,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 89,
+ "drafted": 305,
+ "accepted": 166,
+ "acceptance_rate": 0.5442622950819672,
+ "tokens_per_round": 2.8764044943820224
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 646.3712093265011,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00010608300001990756,
+ "first_s": 0.052181250000018053,
+ "last_s": 1.596157040999998,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00024112500000228465,
+ "first_s": 0.018116458000008606,
+ "last_s": 1.5922712079999997,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00018091600000502694,
+ "first_s": 0.03516187500000001,
+ "last_s": 1.5922490410000023,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 41,
+ "accepted": 32,
+ "acceptance_rate": 0.7804878048780488,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.06923041600001056,
+ "last_s": 1.5861764160000007,
+ "chunks": 223,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 222,
+ "drafted": 40,
+ "accepted": 33,
+ "acceptance_rate": 0.825,
+ "tokens_per_round": 1.1531531531531531
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 815.6355603289662,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0001041659999998501,
+ "first_s": 0.01831516600000782,
+ "last_s": 2.5028167080000117,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0004861250000089967,
+ "first_s": 0.0865736659999925,
+ "last_s": 2.503194707999995,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0003014579999955913,
+ "first_s": 0.052476250000012215,
+ "last_s": 2.502932874999999,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0001798750000148175,
+ "first_s": 0.035417916000000105,
+ "last_s": 2.502847666000008,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0002463749999890297,
+ "first_s": 0.10356395799999518,
+ "last_s": 2.503322166000004,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0003758329999925536,
+ "first_s": 0.0695484160000035,
+ "last_s": 2.5032495410000024,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.000426790999995319,
+ "first_s": 0.12060600000000932,
+ "last_s": 2.507813999999996,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 28,
+ "accepted": 21,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.13762124999999514,
+ "last_s": 2.519432208000012,
+ "chunks": 238,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 237,
+ "drafted": 24,
+ "accepted": 18,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.080168776371308
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.7719859603106,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.37920000012582e-05,
+ "first_s": 0.0356257919999905,
+ "last_s": 5.125112999999999,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0009933340000145563,
+ "first_s": 0.273747458999992,
+ "last_s": 5.126361125000017,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0009274170000139748,
+ "first_s": 0.17177483400001847,
+ "last_s": 5.125810833999992,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00016720900001132577,
+ "first_s": 0.13777945900000077,
+ "last_s": 5.126105499999994,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0005507920000127342,
+ "first_s": 0.06964479199999118,
+ "last_s": 5.125645875000004,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00022087499999656757,
+ "first_s": 0.12081962499999577,
+ "last_s": 5.126271792000011,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0005977090000044427,
+ "first_s": 0.08671187499999178,
+ "last_s": 5.126017750000017,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002775419999920814,
+ "first_s": 0.20584670900001356,
+ "last_s": 5.126156417000004,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00048458399999162793,
+ "first_s": 0.052683625000014445,
+ "last_s": 5.1251904590000095,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0003363750000175969,
+ "first_s": 0.1547320840000168,
+ "last_s": 5.125751459000014,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0008224169999948572,
+ "first_s": 0.23987316700001315,
+ "last_s": 5.125893084000012,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00038716700001373283,
+ "first_s": 0.018572792000014715,
+ "last_s": 5.125086459000016,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0006584589999931723,
+ "first_s": 0.10379062500001623,
+ "last_s": 5.125585125000015,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00043458400000417896,
+ "first_s": 0.25679174999999077,
+ "last_s": 5.126311250000015,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0007168749999948432,
+ "first_s": 0.22288866700000654,
+ "last_s": 5.125942750000007,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.18882725000000278,
+ "last_s": 5.126413417000009,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.08738506794214,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.02016437500000734,
+ "last_s": 0.7342752920000066,
+ "chunks": 92,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 619.427853620994,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0001016669999955866,
+ "first_s": 0.020417124999994485,
+ "last_s": 1.667055375000018,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0002669590000152766,
+ "first_s": 0.03986750000001393,
+ "last_s": 1.66709800000001,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0001857919999963542,
+ "first_s": 0.0787461670000198,
+ "last_s": 1.6670894169999997,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.05925091700001417,
+ "last_s": 1.6670743749999986,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.5509863951829,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 8.554199999366574e-05,
+ "first_s": 0.07897958300000596,
+ "last_s": 2.5782102080000016,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0004218750000006821,
+ "first_s": 0.1565373749999992,
+ "last_s": 2.581858208,
+ "chunks": 256,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 255,
+ "drafted": 6,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ }
+ },
+ {
+ "sent_s": 0.00024970799998413895,
+ "first_s": 0.05960837499998206,
+ "last_s": 2.5779354999999953,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00019904199999132288,
+ "first_s": 0.09835508300000129,
+ "last_s": 2.5783844999999985,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00015183299998966504,
+ "first_s": 0.04022995799999762,
+ "last_s": 2.5778847499999813,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00037075000000186265,
+ "first_s": 0.11774279200000137,
+ "last_s": 2.578158583000004,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00032183299998678194,
+ "first_s": 0.13712483300000144,
+ "last_s": 2.578087457999999,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 3,
+ "accepted": 1,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020816874999979973,
+ "last_s": 2.577838208000003,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 793.0442952546944,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.337500000583532e-05,
+ "first_s": 0.254272708000002,
+ "last_s": 5.1658878749999815,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0005447079999783,
+ "first_s": 0.0990860830000031,
+ "last_s": 5.165940250000006,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00016712499999016472,
+ "first_s": 0.060306166000003714,
+ "last_s": 5.165142000000003,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0005903329999910056,
+ "first_s": 0.040925082999990536,
+ "last_s": 5.165075833000003,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0006879999999966913,
+ "first_s": 0.1572639160000051,
+ "last_s": 5.166023082999999,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00023554099999500977,
+ "first_s": 0.07968662499999368,
+ "last_s": 5.165524790999996,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00028979100000015023,
+ "first_s": 0.021523665999978903,
+ "last_s": 5.165043374999982,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0007354999999904521,
+ "first_s": 0.118454665999991,
+ "last_s": 5.165688208000006,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00044608299998571965,
+ "first_s": 0.292976082999985,
+ "last_s": 5.165848041000004,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0006405000000029304,
+ "first_s": 0.17663833299999965,
+ "last_s": 5.165749874999989,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0003424999999879219,
+ "first_s": 0.3124002909999888,
+ "last_s": 5.166113290999988,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0007830829999875277,
+ "first_s": 0.23488570799997888,
+ "last_s": 5.165363208000002,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0008322079999913967,
+ "first_s": 0.273624749999982,
+ "last_s": 5.1662552079999955,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00039424999999937427,
+ "first_s": 0.19606749999999806,
+ "last_s": 5.166167624999986,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0004940000000033251,
+ "first_s": 0.13784566600000403,
+ "last_s": 5.165808124999984,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.21552279099998373,
+ "last_s": 5.1659797079999805,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 288.3251495763636,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01751966700001617,
+ "last_s": 0.9019378330000052,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 661.7471569888854,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.295800001041243e-05,
+ "first_s": 0.017952000000008184,
+ "last_s": 1.559315708000014,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 63,
+ "accepted": 37,
+ "acceptance_rate": 0.5873015873015873,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.00023954200000275705,
+ "first_s": 0.06702970800000685,
+ "last_s": 1.559305749999993,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.00016483300001368661,
+ "first_s": 0.05068258299999684,
+ "last_s": 1.5593262080000159,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.03432762500000308,
+ "last_s": 1.559288874999993,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 852.361559328986,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.658299998704933e-05,
+ "first_s": 0.13285808300000213,
+ "last_s": 2.4116938330000153,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0004410409999877629,
+ "first_s": 0.08385429099999442,
+ "last_s": 2.4072955409999963,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00017108299999790688,
+ "first_s": 0.10018154100001198,
+ "last_s": 2.407141375000009,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00034004100001538973,
+ "first_s": 0.051151207999993176,
+ "last_s": 2.4069557500000087,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00027695800000060444,
+ "first_s": 0.018343291000007866,
+ "last_s": 2.406800583000006,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00022220799999672636,
+ "first_s": 0.06750845799999183,
+ "last_s": 2.4070947909999916,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00038791599999399295,
+ "first_s": 0.11651170799999022,
+ "last_s": 2.411668249999991,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.034763040999990835,
+ "last_s": 2.4068437079999967,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 849.7301738481642,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.370799998009716e-05,
+ "first_s": 0.08396166599999333,
+ "last_s": 4.810981999999996,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0007197079999912148,
+ "first_s": 0.21460262499999772,
+ "last_s": 4.810679999999991,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00016783299997769063,
+ "first_s": 0.03487445799999023,
+ "last_s": 4.810400125000001,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00034124999999107786,
+ "first_s": 0.051219332999977496,
+ "last_s": 4.810703082999993,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0005149999999787269,
+ "first_s": 0.06757191599999146,
+ "last_s": 4.811300790999979,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0010477909999906387,
+ "first_s": 0.14932912499997997,
+ "last_s": 4.810914957999984,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0006081659999779276,
+ "first_s": 0.23090833300000213,
+ "last_s": 4.811109208000005,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0008241660000010143,
+ "first_s": 0.01852233299999284,
+ "last_s": 4.810266999999982,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00024054099998238598,
+ "first_s": 0.2472579579999774,
+ "last_s": 4.82003070799999,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 49,
+ "accepted": 19,
+ "acceptance_rate": 0.3877551020408163,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.000561665999981642,
+ "first_s": 0.13296404099997972,
+ "last_s": 4.81121458299998,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0002917909999951007,
+ "first_s": 0.18190724999999475,
+ "last_s": 4.810565624999981,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0009509999999863794,
+ "first_s": 0.1656304999999918,
+ "last_s": 4.810872625000002,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0006541660000038974,
+ "first_s": 0.10027454100000455,
+ "last_s": 4.810770290999983,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.000441416000001027,
+ "first_s": 0.11662024999998266,
+ "last_s": 4.810614290999979,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00039400000000000546,
+ "first_s": 0.26357329100000015,
+ "last_s": 4.820046540999982,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.1982730409999931,
+ "last_s": 4.811157332999983,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 260.755637278171,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020178915999991887,
+ "last_s": 0.9981059999999786,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 610.621019530317,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.349999999130887e-05,
+ "first_s": 0.039391499999993584,
+ "last_s": 1.690934124999984,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.00023124999998458406,
+ "first_s": 0.07702774999998496,
+ "last_s": 1.6909574999999961,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.00016170899999679023,
+ "first_s": 0.05818437499999618,
+ "last_s": 1.6909441249999873,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.02052695899999435,
+ "last_s": 1.6909213749999878,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 35,
+ "accepted": 8,
+ "acceptance_rate": 0.22857142857142856,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 810.8527875896573,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.325000002036177e-05,
+ "first_s": 0.15296854200002485,
+ "last_s": 2.536549292000018,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0004953330000034839,
+ "first_s": 0.13413920800002188,
+ "last_s": 2.5364608330000067,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0003479169999991427,
+ "first_s": 0.1153612920000171,
+ "last_s": 2.5362098750000257,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0001584580000155711,
+ "first_s": 0.059039708000000246,
+ "last_s": 2.536182917000019,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00021399999999971442,
+ "first_s": 0.09662441700001523,
+ "last_s": 2.536114000000026,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.000270958000015753,
+ "first_s": 0.04021508300002097,
+ "last_s": 2.5361284170000147,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00039370800001847783,
+ "first_s": 0.0778280830000142,
+ "last_s": 2.5365047920000166,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020679542000010542,
+ "last_s": 2.5360662080000225,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 803.4663797204113,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 8.845800002177384e-05,
+ "first_s": 0.04009933300000057,
+ "last_s": 5.094765332999998,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0008120410000174161,
+ "first_s": 0.17176145800002018,
+ "last_s": 5.094510458000002,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000310708000000659,
+ "first_s": 0.05898112500000252,
+ "last_s": 5.094143791000022,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0001566660000094089,
+ "first_s": 0.24710858299999927,
+ "last_s": 5.094611000000015,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005229580000047918,
+ "first_s": 0.20944224999999506,
+ "last_s": 5.094841625000015,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000207208000006176,
+ "first_s": 0.02121933300000478,
+ "last_s": 5.094091125000006,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0002610000000231594,
+ "first_s": 0.19056554100001222,
+ "last_s": 5.094526875000014,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005700410000031297,
+ "first_s": 0.15298312500001998,
+ "last_s": 5.095028416000019,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007608330000152819,
+ "first_s": 0.3035269160000098,
+ "last_s": 5.099216500000011,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0006172909999975218,
+ "first_s": 0.26590962500000614,
+ "last_s": 5.094901625000006,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003609580000158985,
+ "first_s": 0.07779525000000831,
+ "last_s": 5.094320916000015,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000669291000008343,
+ "first_s": 0.28474433300002033,
+ "last_s": 5.0992055410000034,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 35,
+ "accepted": 3,
+ "acceptance_rate": 0.08571428571428572,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.000412083000014718,
+ "first_s": 0.0965630830000066,
+ "last_s": 5.09512112500002,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0004597080000223741,
+ "first_s": 0.13415579100001196,
+ "last_s": 5.095075790999999,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007149580000032074,
+ "first_s": 0.22830112500000155,
+ "last_s": 5.094542250000018,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.11536295800001994,
+ "last_s": 5.094663875000009,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 357.11403543936194,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.018142707999999175,
+ "last_s": 0.7322003330000086,
+ "chunks": 90,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 89,
+ "drafted": 298,
+ "accepted": 166,
+ "acceptance_rate": 0.5570469798657718,
+ "tokens_per_round": 2.8764044943820224
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 633.0089663616201,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00011112499998944259,
+ "first_s": 0.03549404099999265,
+ "last_s": 1.6050766660000022,
+ "chunks": 221,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 220,
+ "drafted": 53,
+ "accepted": 35,
+ "acceptance_rate": 0.660377358490566,
+ "tokens_per_round": 1.1636363636363636
+ }
+ },
+ {
+ "sent_s": 0.00029341600000520884,
+ "first_s": 0.06967379100001381,
+ "last_s": 1.6297457910000048,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 55,
+ "accepted": 33,
+ "acceptance_rate": 0.6,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0001966250000009495,
+ "first_s": 0.05259558299999867,
+ "last_s": 1.6297132910000016,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 56,
+ "accepted": 33,
+ "acceptance_rate": 0.5892857142857143,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.018394207999989476,
+ "last_s": 1.6050416660000053,
+ "chunks": 221,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 220,
+ "drafted": 54,
+ "accepted": 35,
+ "acceptance_rate": 0.6481481481481481,
+ "tokens_per_round": 1.1636363636363636
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 811.9038369967539,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.791599998720812e-05,
+ "first_s": 0.10382895800000824,
+ "last_s": 2.496403291000007,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 39,
+ "accepted": 30,
+ "acceptance_rate": 0.7692307692307693,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.00037237499998354906,
+ "first_s": 0.018563124999985803,
+ "last_s": 2.483367208000004,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 42,
+ "accepted": 31,
+ "acceptance_rate": 0.7380952380952381,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00030970800000318377,
+ "first_s": 0.035656749999986914,
+ "last_s": 2.4962025410000024,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.00023995800000875533,
+ "first_s": 0.06971212499999524,
+ "last_s": 2.496284457999991,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 42,
+ "accepted": 31,
+ "acceptance_rate": 0.7380952380952381,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.00017233299999475094,
+ "first_s": 0.086754749999983,
+ "last_s": 2.4965703750000046,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 41,
+ "accepted": 31,
+ "acceptance_rate": 0.7560975609756098,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.00047766599999476966,
+ "first_s": 0.05266662500000052,
+ "last_s": 2.496250916000008,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 45,
+ "accepted": 31,
+ "acceptance_rate": 0.6888888888888889,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.000425707999994529,
+ "first_s": 0.13798900000000458,
+ "last_s": 2.5311759580000057,
+ "chunks": 230,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 229,
+ "drafted": 42,
+ "accepted": 26,
+ "acceptance_rate": 0.6190476190476191,
+ "tokens_per_round": 1.1179039301310043
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.12091237499998897,
+ "last_s": 2.465353166,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 48,
+ "accepted": 32,
+ "acceptance_rate": 0.6666666666666666,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 883.3093581379466,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.099999999762076e-05,
+ "first_s": 0.018246500000003607,
+ "last_s": 3.8595087500000034,
+ "chunks": 173,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 172,
+ "drafted": 118,
+ "accepted": 83,
+ "acceptance_rate": 0.7033898305084746,
+ "tokens_per_round": 1.4883720930232558
+ }
+ },
+ {
+ "sent_s": 0.000520624999978736,
+ "first_s": 0.08644045899998787,
+ "last_s": 4.606953958999981,
+ "chunks": 215,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 214,
+ "drafted": 62,
+ "accepted": 41,
+ "acceptance_rate": 0.6612903225806451,
+ "tokens_per_round": 1.1962616822429906
+ }
+ },
+ {
+ "sent_s": 0.0001584999999977299,
+ "first_s": 0.20598312499998883,
+ "last_s": 4.626133334000002,
+ "chunks": 218,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 217,
+ "drafted": 48,
+ "accepted": 38,
+ "acceptance_rate": 0.7916666666666666,
+ "tokens_per_round": 1.1797235023041475
+ }
+ },
+ {
+ "sent_s": 0.00021029199999134107,
+ "first_s": 0.17181783399999517,
+ "last_s": 4.341668666999993,
+ "chunks": 198,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 197,
+ "drafted": 74,
+ "accepted": 58,
+ "acceptance_rate": 0.7837837837837838,
+ "tokens_per_round": 1.299492385786802
+ }
+ },
+ {
+ "sent_s": 0.0009690839999905165,
+ "first_s": 0.27408645899998874,
+ "last_s": 4.429191208999981,
+ "chunks": 204,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 203,
+ "drafted": 62,
+ "accepted": 52,
+ "acceptance_rate": 0.8387096774193549,
+ "tokens_per_round": 1.2610837438423645
+ }
+ },
+ {
+ "sent_s": 0.0005835839999974723,
+ "first_s": 0.22295670899998754,
+ "last_s": 4.626142375000001,
+ "chunks": 218,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 217,
+ "drafted": 46,
+ "accepted": 38,
+ "acceptance_rate": 0.8260869565217391,
+ "tokens_per_round": 1.1797235023041475
+ }
+ },
+ {
+ "sent_s": 0.0002605839999887394,
+ "first_s": 0.06939641699997878,
+ "last_s": 4.598476750000003,
+ "chunks": 214,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 213,
+ "drafted": 60,
+ "accepted": 42,
+ "acceptance_rate": 0.7,
+ "tokens_per_round": 1.2018779342723005
+ }
+ },
+ {
+ "sent_s": 0.0006320839999887085,
+ "first_s": 0.05231562499997722,
+ "last_s": 4.6068307919999825,
+ "chunks": 215,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 214,
+ "drafted": 56,
+ "accepted": 41,
+ "acceptance_rate": 0.7321428571428571,
+ "tokens_per_round": 1.1962616822429906
+ }
+ },
+ {
+ "sent_s": 0.00031029199999466073,
+ "first_s": 0.25705649999997604,
+ "last_s": 4.637239792000003,
+ "chunks": 220,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 219,
+ "drafted": 45,
+ "accepted": 37,
+ "acceptance_rate": 0.8222222222222222,
+ "tokens_per_round": 1.1689497716894977
+ }
+ },
+ {
+ "sent_s": 0.0008008339999889813,
+ "first_s": 0.18887137499999085,
+ "last_s": 4.429128374999976,
+ "chunks": 204,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 203,
+ "drafted": 65,
+ "accepted": 52,
+ "acceptance_rate": 0.8,
+ "tokens_per_round": 1.2610837438423645
+ }
+ },
+ {
+ "sent_s": 0.000749458999990793,
+ "first_s": 0.1376906249999763,
+ "last_s": 4.189068583999983,
+ "chunks": 191,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 190,
+ "drafted": 82,
+ "accepted": 65,
+ "acceptance_rate": 0.7926829268292683,
+ "tokens_per_round": 1.3473684210526315
+ }
+ },
+ {
+ "sent_s": 0.00035833399999773974,
+ "first_s": 0.03527495899999167,
+ "last_s": 4.444412958999976,
+ "chunks": 205,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 204,
+ "drafted": 83,
+ "accepted": 51,
+ "acceptance_rate": 0.6144578313253012,
+ "tokens_per_round": 1.2549019607843137
+ }
+ },
+ {
+ "sent_s": 0.0007010839999850305,
+ "first_s": 0.10352499999999054,
+ "last_s": 4.384193791999991,
+ "chunks": 201,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 200,
+ "drafted": 73,
+ "accepted": 55,
+ "acceptance_rate": 0.7534246575342466,
+ "tokens_per_round": 1.28
+ }
+ },
+ {
+ "sent_s": 0.0004536249999773645,
+ "first_s": 0.23998974999997813,
+ "last_s": 4.620054084000003,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 51,
+ "accepted": 39,
+ "acceptance_rate": 0.7647058823529411,
+ "tokens_per_round": 1.1851851851851851
+ }
+ },
+ {
+ "sent_s": 0.00040704199997776414,
+ "first_s": 0.15474045899998146,
+ "last_s": 4.444532624999994,
+ "chunks": 205,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 204,
+ "drafted": 62,
+ "accepted": 51,
+ "acceptance_rate": 0.8225806451612904,
+ "tokens_per_round": 1.2549019607843137
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.1206221669999934,
+ "last_s": 4.3558931669999765,
+ "chunks": 199,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 198,
+ "drafted": 75,
+ "accepted": 57,
+ "acceptance_rate": 0.76,
+ "tokens_per_round": 1.292929292929293
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 357.13808429747166,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020374333000006573,
+ "last_s": 0.7343838750000202,
+ "chunks": 92,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.2194080955929,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.549999998625935e-05,
+ "first_s": 0.040322750000001406,
+ "last_s": 1.6757240409999952,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 16,
+ "accepted": 6,
+ "acceptance_rate": 0.375,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00024083299999233532,
+ "first_s": 0.07914074999999343,
+ "last_s": 1.675769874999986,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00016987499998322164,
+ "first_s": 0.020515290999981062,
+ "last_s": 1.6757372079999868,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.05971466599999076,
+ "last_s": 1.6757571659999826,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 795.6656503728173,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.137500001088483e-05,
+ "first_s": 0.07913854200000969,
+ "last_s": 2.584796124999997,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 5,
+ "accepted": 2,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.0002701250000143318,
+ "first_s": 0.020917375000010452,
+ "last_s": 2.570523709000014,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0004204170000150498,
+ "first_s": 0.15666095899999277,
+ "last_s": 2.5707824170000038,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 16,
+ "accepted": 5,
+ "acceptance_rate": 0.3125,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0001636670000095819,
+ "first_s": 0.13726049999999645,
+ "last_s": 2.5706546670000137,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 16,
+ "accepted": 5,
+ "acceptance_rate": 0.3125,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.000354375000000573,
+ "first_s": 0.1179348750000031,
+ "last_s": 2.5708748750000154,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 9,
+ "accepted": 5,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0005322920000025988,
+ "first_s": 0.09854445900000997,
+ "last_s": 2.5848083749999944,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 5,
+ "accepted": 2,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.0002184589999956188,
+ "first_s": 0.05977058400000601,
+ "last_s": 2.570543917000009,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.040353542000019615,
+ "last_s": 2.570704667000001,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 873.8662039536016,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 8.758299998135044e-05,
+ "first_s": 0.021479124999984833,
+ "last_s": 4.573498540999992,
+ "chunks": 240,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 29,
+ "accepted": 16,
+ "acceptance_rate": 0.5517241379310345,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00015283299995871857,
+ "first_s": 0.040829290999965906,
+ "last_s": 4.685387665999997,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 19,
+ "accepted": 6,
+ "acceptance_rate": 0.3157894736842105,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0008949999999572356,
+ "first_s": 0.2153574579999713,
+ "last_s": 4.677621249999959,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 15,
+ "accepted": 7,
+ "acceptance_rate": 0.4666666666666667,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0005408750000128748,
+ "first_s": 0.09905920799997148,
+ "last_s": 4.435126749999995,
+ "chunks": 230,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 229,
+ "drafted": 43,
+ "accepted": 26,
+ "acceptance_rate": 0.6046511627906976,
+ "tokens_per_round": 1.1179039301310043
+ }
+ },
+ {
+ "sent_s": 0.0004524579999838352,
+ "first_s": 0.060291582999980164,
+ "last_s": 4.690369165999982,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 15,
+ "accepted": 5,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0007604580000020178,
+ "first_s": 0.2928932080000095,
+ "last_s": 4.685423915999991,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.000204957999983435,
+ "first_s": 0.1572447499999612,
+ "last_s": 4.450082915999985,
+ "chunks": 231,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 230,
+ "drafted": 41,
+ "accepted": 25,
+ "acceptance_rate": 0.6097560975609756,
+ "tokens_per_round": 1.1130434782608696
+ }
+ },
+ {
+ "sent_s": 0.0005966249999573847,
+ "first_s": 0.31224591599999485,
+ "last_s": 4.685437250000007,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00025804099999504615,
+ "first_s": 0.07968004099996051,
+ "last_s": 4.506589874999975,
+ "chunks": 235,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 36,
+ "accepted": 21,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0007083749999878819,
+ "first_s": 0.2734678329999838,
+ "last_s": 4.690386082999964,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 12,
+ "accepted": 5,
+ "acceptance_rate": 0.4166666666666667,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0003562499999816282,
+ "first_s": 0.23470574999998917,
+ "last_s": 4.677773915999978,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 16,
+ "accepted": 7,
+ "acceptance_rate": 0.4375,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00030695800001012685,
+ "first_s": 0.1378970409999738,
+ "last_s": 3.5230339999999956,
+ "chunks": 172,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 171,
+ "drafted": 111,
+ "accepted": 84,
+ "acceptance_rate": 0.7567567567567568,
+ "tokens_per_round": 1.4970760233918128
+ }
+ },
+ {
+ "sent_s": 0.0006464999999593601,
+ "first_s": 0.25408454099999744,
+ "last_s": 4.67772920799996,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 15,
+ "accepted": 7,
+ "acceptance_rate": 0.4666666666666667,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0010505409999836957,
+ "first_s": 0.19596420799996395,
+ "last_s": 4.6674600000000055,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 21,
+ "accepted": 8,
+ "acceptance_rate": 0.38095238095238093,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.0004032909999978074,
+ "first_s": 0.11851341599998477,
+ "last_s": 4.645478665999974,
+ "chunks": 246,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 245,
+ "drafted": 18,
+ "accepted": 10,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.0448979591836736
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.17660416600000417,
+ "last_s": 4.598797999999988,
+ "chunks": 242,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 241,
+ "drafted": 28,
+ "accepted": 14,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.062240663900415
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 290.1445709497884,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.017870000000016262,
+ "last_s": 0.8967422090000241,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 652.1462364408767,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.841700000379205e-05,
+ "first_s": 0.03442879199997151,
+ "last_s": 1.5783031249999908,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0002859169999851474,
+ "first_s": 0.06710795800000824,
+ "last_s": 1.582126833000018,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00020595799998091024,
+ "first_s": 0.050818999999989956,
+ "last_s": 1.5783118329999866,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.018060457999979462,
+ "last_s": 1.5782788750000236,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 830.8257532362089,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.612500002731394e-05,
+ "first_s": 0.0183462080000254,
+ "last_s": 2.465988500000037,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0004637080000406968,
+ "first_s": 0.11651212499998564,
+ "last_s": 2.4664302909999947,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00017512499999838838,
+ "first_s": 0.13292641600003208,
+ "last_s": 2.473734707999995,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.00037058300000580857,
+ "first_s": 0.051134208000007675,
+ "last_s": 2.466002875000015,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0002378330000283313,
+ "first_s": 0.034738708000020324,
+ "last_s": 2.4660924579999914,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00041770799998630537,
+ "first_s": 0.08384120800002393,
+ "last_s": 2.4662536250000358,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0002934999999979482,
+ "first_s": 0.10015804099998604,
+ "last_s": 2.4664831250000248,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.06747904100001278,
+ "last_s": 2.4663362500000403,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 830.1942250072074,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 8.816700000124911e-05,
+ "first_s": 0.035117709000019204,
+ "last_s": 4.9320778750000045,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00105195899999444,
+ "first_s": 0.14947258400002283,
+ "last_s": 4.933139250000011,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00015245899999172252,
+ "first_s": 0.08419329200000902,
+ "last_s": 4.932175000000029,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00020837499999970532,
+ "first_s": 0.10050216700000192,
+ "last_s": 4.932412166999995,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00042595899998332243,
+ "first_s": 0.21482041699999854,
+ "last_s": 4.9328578340000035,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0008812500000203727,
+ "first_s": 0.24742183400002205,
+ "last_s": 4.932897459000003,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0005659590000277603,
+ "first_s": 0.23112195900000643,
+ "last_s": 4.933094625000024,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00027537500000107684,
+ "first_s": 0.19848766699999487,
+ "last_s": 4.932697791999999,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00032454199998710465,
+ "first_s": 0.16584158400002025,
+ "last_s": 4.933056958999998,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.000725792000025649,
+ "first_s": 0.018690292000030695,
+ "last_s": 4.932056584000009,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0006149999999820466,
+ "first_s": 0.26380495900002643,
+ "last_s": 4.93281241699998,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0004815419999886217,
+ "first_s": 0.11685229199997593,
+ "last_s": 4.932662250000021,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00037687500002903107,
+ "first_s": 0.06783812499998021,
+ "last_s": 4.9330156670000065,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.000664792000009129,
+ "first_s": 0.18216512499998316,
+ "last_s": 4.933202917000017,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0008210000000303808,
+ "first_s": 0.13316987500002142,
+ "last_s": 4.932942249999996,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.05148687500002325,
+ "last_s": 4.932089542000028,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 262.17824367823357,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020563792000018566,
+ "last_s": 0.9931845419999945,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 623.428583696316,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.27079999542002e-05,
+ "first_s": 0.0769434999999703,
+ "last_s": 1.6566039159999946,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00023712499995554026,
+ "first_s": 0.05810608299998421,
+ "last_s": 1.656582125,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00015891599997530648,
+ "first_s": 0.020490290999987337,
+ "last_s": 1.65656974999996,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.03932337499998084,
+ "last_s": 1.6565929159999655,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 804.3767029105711,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.345799998072835e-05,
+ "first_s": 0.0781672499999786,
+ "last_s": 2.553240374999973,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005230419999975311,
+ "first_s": 0.13456129200000078,
+ "last_s": 2.553431499999988,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000334582999983013,
+ "first_s": 0.05930791700001237,
+ "last_s": 2.5531172919999676,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0001665830000092683,
+ "first_s": 0.04046745799996643,
+ "last_s": 2.5531376249999767,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0004385419999834994,
+ "first_s": 0.1533778329999791,
+ "last_s": 2.557074666999995,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002247919999831538,
+ "first_s": 0.020949499999971977,
+ "last_s": 2.5531034580000096,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00027691700000787023,
+ "first_s": 0.11573950000001787,
+ "last_s": 2.553477832999988,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.09695012499997802,
+ "last_s": 2.55359920799998,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 1,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 799.5577348294199,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.379199997283649e-05,
+ "first_s": 0.020883708999974715,
+ "last_s": 5.122531833999972,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0009131669999646874,
+ "first_s": 0.2662117919999787,
+ "last_s": 5.123171499999955,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0001618750000034197,
+ "first_s": 0.15336545899998555,
+ "last_s": 5.123476541999992,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00021079199996165698,
+ "first_s": 0.2473814169999855,
+ "last_s": 5.123264333999998,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0004082919999746082,
+ "first_s": 0.059267333999969196,
+ "last_s": 5.1225703749999525,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0004592919999595324,
+ "first_s": 0.22860629199999494,
+ "last_s": 5.123352499999953,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.001210374999971009,
+ "first_s": 0.28501837499999283,
+ "last_s": 5.123573999999962,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006401669999718251,
+ "first_s": 0.04041541699996287,
+ "last_s": 5.122544374999961,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00026254199997310934,
+ "first_s": 0.0780780839999693,
+ "last_s": 5.122579916999996,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005188749999547326,
+ "first_s": 0.11574570899995251,
+ "last_s": 5.1237047089999805,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005762089999734599,
+ "first_s": 0.1721516249999695,
+ "last_s": 5.123214166999958,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003622919999770602,
+ "first_s": 0.3038521249999917,
+ "last_s": 5.1233894169999985,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007301669999719707,
+ "first_s": 0.20979229199997462,
+ "last_s": 5.12298283399997,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003171669999915139,
+ "first_s": 0.13458849999994982,
+ "last_s": 5.122833166999953,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006837499999505781,
+ "first_s": 0.09692954199999804,
+ "last_s": 5.1231065419999595,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.19095787499998096,
+ "last_s": 5.123651749999965,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 288.80212241245,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019138124999983575,
+ "last_s": 0.9020956250000154,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 662.1169568286863,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00010299999996732367,
+ "first_s": 0.05080774999998994,
+ "last_s": 1.5586303339999859,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.00024045899999691756,
+ "first_s": 0.06717550000001893,
+ "last_s": 1.5586626670000214,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.00018233399998734967,
+ "first_s": 0.018149333999986084,
+ "last_s": 1.5586120840000035,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 63,
+ "accepted": 37,
+ "acceptance_rate": 0.5873015873015873,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.03449033399999735,
+ "last_s": 1.5586387920000107,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 63,
+ "accepted": 37,
+ "acceptance_rate": 0.5873015873015873,
+ "tokens_per_round": 1.1743119266055047
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 851.9150784740538,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.316599999920072e-05,
+ "first_s": 0.017899332999945727,
+ "last_s": 2.4075728329999606,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00015991599997278172,
+ "first_s": 0.13227741599996534,
+ "last_s": 2.4124968329999774,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0004250409999713156,
+ "first_s": 0.11592979099998502,
+ "last_s": 2.412504207999973,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00021020799999860174,
+ "first_s": 0.034299665999981244,
+ "last_s": 2.4076144999999656,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0003744579999533926,
+ "first_s": 0.08332545799999025,
+ "last_s": 2.4077246249999575,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00026279099995463184,
+ "first_s": 0.0670137079999904,
+ "last_s": 2.407703332999972,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.000328249999995478,
+ "first_s": 0.09966033299997434,
+ "last_s": 2.407870832999947,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.0506086249999953,
+ "last_s": 2.407634250000001,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 849.8077382579876,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.60000000418404e-05,
+ "first_s": 0.21488825000000134,
+ "last_s": 4.810995583000022,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0009846660000221164,
+ "first_s": 0.2311888330000329,
+ "last_s": 4.810556125000005,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0006709160000468728,
+ "first_s": 0.10054408300004525,
+ "last_s": 4.81074537500001,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0004253750000202672,
+ "first_s": 0.14955316600003243,
+ "last_s": 4.810960583000053,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00016945800001622047,
+ "first_s": 0.1168898330000161,
+ "last_s": 4.810351666000031,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.000221833000011884,
+ "first_s": 0.018796790999999757,
+ "last_s": 4.810066791000054,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00027404100001149345,
+ "first_s": 0.0678322080000271,
+ "last_s": 4.810110416000043,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0005349160000491793,
+ "first_s": 0.1985286250000513,
+ "last_s": 4.81078954100002,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00032766600003242274,
+ "first_s": 0.08417212500000915,
+ "last_s": 4.810636458000033,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0004727500000285545,
+ "first_s": 0.16590491600004498,
+ "last_s": 4.810835791000045,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0008189580000248498,
+ "first_s": 0.18221120800001245,
+ "last_s": 4.810424791000003,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0007290830000101778,
+ "first_s": 0.05148583300001519,
+ "last_s": 4.810153916000047,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.000581375000024309,
+ "first_s": 0.247501375000013,
+ "last_s": 4.8198596660000135,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 49,
+ "accepted": 19,
+ "acceptance_rate": 0.3877551020408163,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.0003775830000449787,
+ "first_s": 0.03516545800005133,
+ "last_s": 4.81009825000001,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0006237910000095326,
+ "first_s": 0.1332368750000228,
+ "last_s": 4.810877625000046,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.2638208750000217,
+ "last_s": 4.819882750000033,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 261.43083917633663,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020143625000002885,
+ "last_s": 0.9955449999999928,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 610.2119110482674,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.208300002683245e-05,
+ "first_s": 0.05877595800001245,
+ "last_s": 1.6919833750000066,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.00024008299999422889,
+ "first_s": 0.03996425000002546,
+ "last_s": 1.691958999999997,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 35,
+ "accepted": 8,
+ "acceptance_rate": 0.22857142857142856,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.00016249999998763087,
+ "first_s": 0.07758241599998428,
+ "last_s": 1.6919710829999985,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020432916000004298,
+ "last_s": 1.6919316249999952,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 35,
+ "accepted": 8,
+ "acceptance_rate": 0.22857142857142856,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 810.1186198318942,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 8.63340000023527e-05,
+ "first_s": 0.05907720900000868,
+ "last_s": 2.538353500000028,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0004186670000194681,
+ "first_s": 0.15300479200004702,
+ "last_s": 2.5388381250000407,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00015108400003782663,
+ "first_s": 0.13421100000005026,
+ "last_s": 2.5387360000000285,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0004976250000368054,
+ "first_s": 0.0779480840000133,
+ "last_s": 2.5384655000000294,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0002074170000128106,
+ "first_s": 0.096706834000031,
+ "last_s": 2.5384058340000024,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00025504200004888844,
+ "first_s": 0.02074487500004807,
+ "last_s": 2.538339334,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00032470900004000214,
+ "first_s": 0.11547200000001112,
+ "last_s": 2.5388946250000117,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.040278709000006074,
+ "last_s": 2.5383672090000005,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 802.9041844105632,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 8.9749999972355e-05,
+ "first_s": 0.22871504199997617,
+ "last_s": 5.0983305829999495,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007280419999915466,
+ "first_s": 0.20994429199998876,
+ "last_s": 5.098495624999998,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0001584999999977299,
+ "first_s": 0.04069987499997296,
+ "last_s": 5.097708082999986,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0002105829999550224,
+ "first_s": 0.05952895799998714,
+ "last_s": 5.09855941699999,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0012114999999539577,
+ "first_s": 0.28504066699997566,
+ "last_s": 5.1027350829999705,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 35,
+ "accepted": 3,
+ "acceptance_rate": 0.08571428571428572,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002629579999506859,
+ "first_s": 0.1158755419999693,
+ "last_s": 5.097940249999965,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000671291999992718,
+ "first_s": 0.09708154199995533,
+ "last_s": 5.0980255829999805,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007773329999736234,
+ "first_s": 0.15349004199998717,
+ "last_s": 5.097905582999999,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006199999999694228,
+ "first_s": 0.30379487499999414,
+ "last_s": 5.102750791999995,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 31,
+ "accepted": 3,
+ "acceptance_rate": 0.0967741935483871,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00041112499997097984,
+ "first_s": 0.2474956249999991,
+ "last_s": 5.098159291999991,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003143749999594547,
+ "first_s": 0.17228099999999813,
+ "last_s": 5.098092916999974,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00036224999996647966,
+ "first_s": 0.26624462499995616,
+ "last_s": 5.098614749999967,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005654579999827547,
+ "first_s": 0.1346644999999853,
+ "last_s": 5.098270832999958,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0009518329999878006,
+ "first_s": 0.0783074579999834,
+ "last_s": 5.097780166999996,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00045808299995542257,
+ "first_s": 0.021197999999969852,
+ "last_s": 5.097598374999961,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.19109045799996238,
+ "last_s": 5.098209707999956,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 357.29358499460534,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01779850000002625,
+ "last_s": 0.7314972920000287,
+ "chunks": 90,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 89,
+ "drafted": 298,
+ "accepted": 166,
+ "acceptance_rate": 0.5570469798657718,
+ "tokens_per_round": 2.8764044943820224
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 640.4949813535418,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0001032499999951142,
+ "first_s": 0.0693399589999899,
+ "last_s": 1.605799042000001,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 47,
+ "accepted": 32,
+ "acceptance_rate": 0.6808510638297872,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00022908400001142581,
+ "first_s": 0.03524716700002273,
+ "last_s": 1.6107129170000007,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 50,
+ "accepted": 31,
+ "acceptance_rate": 0.62,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00017433400000754773,
+ "first_s": 0.05225150000001122,
+ "last_s": 1.6057841250000138,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 48,
+ "accepted": 32,
+ "acceptance_rate": 0.6666666666666666,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.018194584000013947,
+ "last_s": 1.6107005840000284,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 51,
+ "accepted": 31,
+ "acceptance_rate": 0.6078431372549019,
+ "tokens_per_round": 1.1428571428571428
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 810.8186930899934,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.645800002999749e-05,
+ "first_s": 0.038159375000020646,
+ "last_s": 2.49951016600005,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 47,
+ "accepted": 31,
+ "acceptance_rate": 0.6595744680851063,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0004860829999984162,
+ "first_s": 0.12334808299999622,
+ "last_s": 2.4685120830000074,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 47,
+ "accepted": 32,
+ "acceptance_rate": 0.6808510638297872,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00017500000001291482,
+ "first_s": 0.1062772500000051,
+ "last_s": 2.4997072500000286,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 39,
+ "accepted": 30,
+ "acceptance_rate": 0.7692307692307693,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0003515000000220425,
+ "first_s": 0.07219908300004363,
+ "last_s": 2.499624666000045,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 45,
+ "accepted": 31,
+ "acceptance_rate": 0.6888888888888889,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0004189160000009906,
+ "first_s": 0.140390291000017,
+ "last_s": 2.5344794160000106,
+ "chunks": 230,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 229,
+ "drafted": 44,
+ "accepted": 26,
+ "acceptance_rate": 0.5909090909090909,
+ "tokens_per_round": 1.1179039301310043
+ }
+ },
+ {
+ "sent_s": 0.00022383300000683448,
+ "first_s": 0.055192207999994025,
+ "last_s": 2.4995872080000368,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0002739580000366004,
+ "first_s": 0.08921379100002014,
+ "last_s": 2.4997978330000024,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 41,
+ "accepted": 31,
+ "acceptance_rate": 0.7560975609756098,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.018503875000021708,
+ "last_s": 2.486560291000046,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 42,
+ "accepted": 31,
+ "acceptance_rate": 0.7380952380952381,
+ "tokens_per_round": 1.1428571428571428
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 896.9609675675463,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.354200000188939e-05,
+ "first_s": 0.13778716699999904,
+ "last_s": 4.558449792000033,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 48,
+ "accepted": 31,
+ "acceptance_rate": 0.6458333333333334,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0001641250000261607,
+ "first_s": 0.22284575000003315,
+ "last_s": 4.567311417000042,
+ "chunks": 227,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 226,
+ "drafted": 44,
+ "accepted": 29,
+ "acceptance_rate": 0.6590909090909091,
+ "tokens_per_round": 1.1327433628318584
+ }
+ },
+ {
+ "sent_s": 0.0007750420000434133,
+ "first_s": 0.03567437500004189,
+ "last_s": 4.558373334000009,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 47,
+ "accepted": 31,
+ "acceptance_rate": 0.6595744680851063,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00021229200001471327,
+ "first_s": 0.15483083399999487,
+ "last_s": 4.032109875000003,
+ "chunks": 194,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 193,
+ "drafted": 80,
+ "accepted": 62,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.3264248704663213
+ }
+ },
+ {
+ "sent_s": 0.0005378750000204491,
+ "first_s": 0.2058414589999984,
+ "last_s": 4.312854167000012,
+ "chunks": 210,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 209,
+ "drafted": 62,
+ "accepted": 46,
+ "acceptance_rate": 0.7419354838709677,
+ "tokens_per_round": 1.2248803827751196
+ }
+ },
+ {
+ "sent_s": 0.0006347089999962918,
+ "first_s": 0.2398775419999879,
+ "last_s": 4.220047916999988,
+ "chunks": 204,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 203,
+ "drafted": 70,
+ "accepted": 52,
+ "acceptance_rate": 0.7428571428571429,
+ "tokens_per_round": 1.2610837438423645
+ }
+ },
+ {
+ "sent_s": 0.0002655420000223785,
+ "first_s": 0.1888389590000088,
+ "last_s": 4.5432662090000235,
+ "chunks": 223,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 222,
+ "drafted": 46,
+ "accepted": 33,
+ "acceptance_rate": 0.717391304347826,
+ "tokens_per_round": 1.1531531531531531
+ }
+ },
+ {
+ "sent_s": 0.000589459000025272,
+ "first_s": 0.2568938340000386,
+ "last_s": 4.293403000000012,
+ "chunks": 209,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 208,
+ "drafted": 63,
+ "accepted": 47,
+ "acceptance_rate": 0.746031746031746,
+ "tokens_per_round": 1.2307692307692308
+ }
+ },
+ {
+ "sent_s": 0.0003151250000428263,
+ "first_s": 0.05272545899998704,
+ "last_s": 4.525703000000021,
+ "chunks": 221,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 220,
+ "drafted": 50,
+ "accepted": 35,
+ "acceptance_rate": 0.7,
+ "tokens_per_round": 1.1636363636363636
+ }
+ },
+ {
+ "sent_s": 0.0008679169999936676,
+ "first_s": 0.2739419590000125,
+ "last_s": 4.515567583999996,
+ "chunks": 220,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 219,
+ "drafted": 49,
+ "accepted": 36,
+ "acceptance_rate": 0.7346938775510204,
+ "tokens_per_round": 1.1689497716894977
+ }
+ },
+ {
+ "sent_s": 0.00036395900002617054,
+ "first_s": 0.12073566700001948,
+ "last_s": 4.535177583999996,
+ "chunks": 222,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 221,
+ "drafted": 52,
+ "accepted": 34,
+ "acceptance_rate": 0.6538461538461539,
+ "tokens_per_round": 1.158371040723982
+ }
+ },
+ {
+ "sent_s": 0.0006935420000218073,
+ "first_s": 0.08665825000002769,
+ "last_s": 4.18508970900001,
+ "chunks": 202,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 201,
+ "drafted": 78,
+ "accepted": 54,
+ "acceptance_rate": 0.6923076923076923,
+ "tokens_per_round": 1.2736318407960199
+ }
+ },
+ {
+ "sent_s": 0.0004630420000353297,
+ "first_s": 0.06971445900001072,
+ "last_s": 4.563319917000001,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 47,
+ "accepted": 30,
+ "acceptance_rate": 0.6382978723404256,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0009719590000258904,
+ "first_s": 0.10370554200000015,
+ "last_s": 4.501289917000008,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.00041558399999530593,
+ "first_s": 0.17182487500002708,
+ "last_s": 4.355611875000022,
+ "chunks": 212,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 211,
+ "drafted": 60,
+ "accepted": 44,
+ "acceptance_rate": 0.7333333333333333,
+ "tokens_per_round": 1.2132701421800949
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.018618500000002314,
+ "last_s": 4.558345916999997,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 50,
+ "accepted": 31,
+ "acceptance_rate": 0.62,
+ "tokens_per_round": 1.1428571428571428
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.86650252228407,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01999879200002397,
+ "last_s": 0.7345517079999695,
+ "chunks": 92,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 616.5100175053622,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.00010187499998437488,
+ "first_s": 0.07893824999996468,
+ "last_s": 1.6751000419999968,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0002444589999868185,
+ "first_s": 0.059547833999999966,
+ "last_s": 1.6750071249999792,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00016962500001227454,
+ "first_s": 0.020625709000000825,
+ "last_s": 1.6749675419999903,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.040122209000003295,
+ "last_s": 1.6749933749999855,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.7632013621845,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.295799998199072e-05,
+ "first_s": 0.13753887500001838,
+ "last_s": 2.5673007080000048,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 16,
+ "accepted": 5,
+ "acceptance_rate": 0.3125,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00043887499998618296,
+ "first_s": 0.09879179199998589,
+ "last_s": 2.581307667000033,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 5,
+ "accepted": 2,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.00016079200003105143,
+ "first_s": 0.06006941700002244,
+ "last_s": 2.567203917000029,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00021183299998028815,
+ "first_s": 0.04066891700000497,
+ "last_s": 2.566976082999986,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00026820800002269607,
+ "first_s": 0.11815679200003615,
+ "last_s": 2.567093457999988,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 9,
+ "accepted": 5,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0003361670000003869,
+ "first_s": 0.15692362499999035,
+ "last_s": 2.567255792000026,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 16,
+ "accepted": 5,
+ "acceptance_rate": 0.3125,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0003837919999796213,
+ "first_s": 0.07947441700002855,
+ "last_s": 2.5812740000000076,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 5,
+ "accepted": 2,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020948458000020764,
+ "last_s": 2.566939208000008,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 857.1512455202633,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.354200000188939e-05,
+ "first_s": 0.09920666699997582,
+ "last_s": 4.723476749999975,
+ "chunks": 247,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 246,
+ "drafted": 22,
+ "accepted": 9,
+ "acceptance_rate": 0.4090909090909091,
+ "tokens_per_round": 1.0406504065040652
+ }
+ },
+ {
+ "sent_s": 0.00015879199997925753,
+ "first_s": 0.0604097089999982,
+ "last_s": 4.774901166999996,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 14,
+ "accepted": 4,
+ "acceptance_rate": 0.2857142857142857,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007706669999834048,
+ "first_s": 0.1767391669999938,
+ "last_s": 4.690660833999971,
+ "chunks": 244,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 243,
+ "drafted": 24,
+ "accepted": 12,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0534979423868314
+ }
+ },
+ {
+ "sent_s": 0.00020845899996402295,
+ "first_s": 0.07982154199999059,
+ "last_s": 4.233565124999984,
+ "chunks": 211,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 210,
+ "drafted": 66,
+ "accepted": 45,
+ "acceptance_rate": 0.6818181818181818,
+ "tokens_per_round": 1.2190476190476192
+ }
+ },
+ {
+ "sent_s": 0.0010249589999489217,
+ "first_s": 0.11857479199994714,
+ "last_s": 4.39235599999995,
+ "chunks": 222,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 221,
+ "drafted": 50,
+ "accepted": 34,
+ "acceptance_rate": 0.68,
+ "tokens_per_round": 1.158371040723982
+ }
+ },
+ {
+ "sent_s": 0.0002622089999704258,
+ "first_s": 0.234921791999966,
+ "last_s": 4.767315209000003,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 11,
+ "accepted": 5,
+ "acceptance_rate": 0.45454545454545453,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0005217919999722653,
+ "first_s": 0.2930434169999785,
+ "last_s": 4.774883041999999,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 7,
+ "accepted": 4,
+ "acceptance_rate": 0.5714285714285714,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000560291999988749,
+ "first_s": 0.19614500000000135,
+ "last_s": 4.781044624999993,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 13,
+ "accepted": 3,
+ "acceptance_rate": 0.23076923076923078,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0004085420000023987,
+ "first_s": 0.021137416999977177,
+ "last_s": 3.7824979589999543,
+ "chunks": 182,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 181,
+ "drafted": 104,
+ "accepted": 74,
+ "acceptance_rate": 0.7115384615384616,
+ "tokens_per_round": 1.4143646408839778
+ }
+ },
+ {
+ "sent_s": 0.0009459999999990032,
+ "first_s": 0.2543154999999615,
+ "last_s": 4.767446791999987,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 11,
+ "accepted": 5,
+ "acceptance_rate": 0.45454545454545453,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0006212089999735326,
+ "first_s": 0.13793583399996123,
+ "last_s": 4.666214541999977,
+ "chunks": 242,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 241,
+ "drafted": 24,
+ "accepted": 14,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.062240663900415
+ }
+ },
+ {
+ "sent_s": 0.00031029199999466073,
+ "first_s": 0.1573439999999664,
+ "last_s": 4.678867708999974,
+ "chunks": 243,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 242,
+ "drafted": 29,
+ "accepted": 13,
+ "acceptance_rate": 0.4482758620689655,
+ "tokens_per_round": 1.0578512396694215
+ }
+ },
+ {
+ "sent_s": 0.00035820899995542277,
+ "first_s": 0.3124601249999728,
+ "last_s": 4.781090833999997,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 12,
+ "accepted": 3,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.000694291999991492,
+ "first_s": 0.2155093339999894,
+ "last_s": 4.781077083999946,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 10,
+ "accepted": 3,
+ "acceptance_rate": 0.3,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0004538339999839991,
+ "first_s": 0.2736370839999722,
+ "last_s": 4.767494124999985,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 9,
+ "accepted": 5,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.04105074999995395,
+ "last_s": 4.173923208999952,
+ "chunks": 207,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 206,
+ "drafted": 73,
+ "accepted": 49,
+ "acceptance_rate": 0.6712328767123288,
+ "tokens_per_round": 1.2427184466019416
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 289.230570484437,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01927791699995396,
+ "last_s": 0.900927458999945,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 651.5197726117417,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.487500000204818e-05,
+ "first_s": 0.018004292000000532,
+ "last_s": 1.5797410420000233,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00025370800000246163,
+ "first_s": 0.05079220800001849,
+ "last_s": 1.5797607499999913,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00016833299997642825,
+ "first_s": 0.03436199999998735,
+ "last_s": 1.5797775830000091,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.06716679200002318,
+ "last_s": 1.5835745829999723,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 829.9116850904256,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.608400000615802e-05,
+ "first_s": 0.051229583999997885,
+ "last_s": 2.468868917000009,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0004387089999795535,
+ "first_s": 0.08406691700002966,
+ "last_s": 2.4689808340000354,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00017216700001654317,
+ "first_s": 0.03484574999998813,
+ "last_s": 2.4688760420000335,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00027787499999476495,
+ "first_s": 0.11678216699999666,
+ "last_s": 2.469216249999988,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0003801670000029844,
+ "first_s": 0.13312216700001045,
+ "last_s": 2.476538667,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.0004904589999910058,
+ "first_s": 0.10042066699998031,
+ "last_s": 2.46926479199999,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0002266249999820502,
+ "first_s": 0.06765316700000312,
+ "last_s": 2.4690102090000323,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01844579200002272,
+ "last_s": 2.4688550420000297,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 828.7649555945501,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.304200000315177e-05,
+ "first_s": 0.1984182499999747,
+ "last_s": 4.941143875000023,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0005334589999961281,
+ "first_s": 0.16576958399997466,
+ "last_s": 4.940930334000029,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00016395900001953123,
+ "first_s": 0.06765066700000943,
+ "last_s": 4.9407357499999875,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0008591669999873375,
+ "first_s": 0.05131333399998539,
+ "last_s": 4.940530000000024,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0002123749999896063,
+ "first_s": 0.018543667000017194,
+ "last_s": 4.940416625000012,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0002662499999814827,
+ "first_s": 0.24748233399998298,
+ "last_s": 4.941041874999996,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0005948749999902248,
+ "first_s": 0.11670366700002432,
+ "last_s": 4.941234375000022,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00031779199997572505,
+ "first_s": 0.1003815839999902,
+ "last_s": 4.940843374999986,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0009180420000234335,
+ "first_s": 0.18208241700000372,
+ "last_s": 4.941531750000024,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0006782500000213076,
+ "first_s": 0.08403616700002203,
+ "last_s": 4.940987167000003,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00036941700000170385,
+ "first_s": 0.03497149999998328,
+ "last_s": 4.940430166999988,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0004694169999766018,
+ "first_s": 0.21478429199999027,
+ "last_s": 4.941419459000031,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0007815839999807395,
+ "first_s": 0.23112329200000659,
+ "last_s": 4.941494583999997,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0009819169999900623,
+ "first_s": 0.26384345899998607,
+ "last_s": 4.941331084000012,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.00041783400001804694,
+ "first_s": 0.14942258399997854,
+ "last_s": 4.941079417000026,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.13306408400001146,
+ "last_s": 4.941375124999979,
+ "chunks": 240,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 239,
+ "drafted": 19,
+ "accepted": 16,
+ "acceptance_rate": 0.8421052631578947,
+ "tokens_per_round": 1.0711297071129706
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 262.4367240358528,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020232375000034608,
+ "last_s": 0.9918951669999956,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 622.7480552674332,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.020900000678012e-05,
+ "first_s": 0.0393185840000001,
+ "last_s": 1.658318917000031,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00021545900000319307,
+ "first_s": 0.02044062500004884,
+ "last_s": 1.658307667000031,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0001620840000100543,
+ "first_s": 0.058112750000020696,
+ "last_s": 1.658328417000007,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.0769772500000272,
+ "last_s": 1.6583421670000007,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 803.714540097247,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 8.941600003709027e-05,
+ "first_s": 0.09670395800003462,
+ "last_s": 2.5554197499999987,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003854580000393071,
+ "first_s": 0.11549291599999378,
+ "last_s": 2.5553644160000317,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00025637500004904723,
+ "first_s": 0.13432833300004177,
+ "last_s": 2.5555472080000072,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00020395800004280318,
+ "first_s": 0.07788545800002566,
+ "last_s": 2.5549879580000265,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00015408300004082776,
+ "first_s": 0.020731458000000202,
+ "last_s": 2.554933833000007,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0004990410000118572,
+ "first_s": 0.15311308299999382,
+ "last_s": 2.5589460830000235,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00032250000003841706,
+ "first_s": 0.04028633300004003,
+ "last_s": 2.5549463750000427,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.05905616600000485,
+ "last_s": 2.554976666000016,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 797.5772223478886,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.662500002605157e-05,
+ "first_s": 0.21002141600001778,
+ "last_s": 5.1364576660000125,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007576250000056461,
+ "first_s": 0.24770233300000655,
+ "last_s": 5.1359366660000205,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00016866600003595522,
+ "first_s": 0.07852545800000144,
+ "last_s": 5.1356096659999935,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0008556660000067495,
+ "first_s": 0.1537516250000408,
+ "last_s": 5.1361924160000285,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0002256250000414184,
+ "first_s": 0.17250141600004554,
+ "last_s": 5.13665770800003,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005367080000269198,
+ "first_s": 0.2665476660000081,
+ "last_s": 5.136063791000026,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0002780410000013944,
+ "first_s": 0.0211655830000268,
+ "last_s": 5.135473832999992,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005808330000149908,
+ "first_s": 0.28539037500001996,
+ "last_s": 5.136316791000013,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00033262499999864303,
+ "first_s": 0.09731941600000482,
+ "last_s": 5.136251458000004,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003818750000164073,
+ "first_s": 0.3042443749999961,
+ "last_s": 5.136134208000044,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0009234999999989668,
+ "first_s": 0.22887383300002284,
+ "last_s": 5.136615790999997,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0010432910000304219,
+ "first_s": 0.11616412500001161,
+ "last_s": 5.136565625000003,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000431083000023591,
+ "first_s": 0.19128108300003532,
+ "last_s": 5.135753458000011,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006715830000416645,
+ "first_s": 0.1349400830000036,
+ "last_s": 5.136520415999996,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00047654100001182087,
+ "first_s": 0.04082654100000127,
+ "last_s": 5.135484958000006,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.0596649160000311,
+ "last_s": 5.135501207999994,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.76141662801433,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019272375000014108,
+ "last_s": 0.7360448750000046,
+ "chunks": 90,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 89,
+ "drafted": 298,
+ "accepted": 166,
+ "acceptance_rate": 0.5570469798657718,
+ "tokens_per_round": 2.8764044943820224
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.2786322623483,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00010016699997095202,
+ "first_s": 0.01819808400000511,
+ "last_s": 1.5950289589999898,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0002579169999989972,
+ "first_s": 0.05222191699999712,
+ "last_s": 1.5989105840000093,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00017329199999949196,
+ "first_s": 0.0692620419999912,
+ "last_s": 1.5889688340000134,
+ "chunks": 223,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 222,
+ "drafted": 40,
+ "accepted": 33,
+ "acceptance_rate": 0.825,
+ "tokens_per_round": 1.1531531531531531
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.03522750000001906,
+ "last_s": 1.5950382089999948,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 41,
+ "accepted": 32,
+ "acceptance_rate": 0.7804878048780488,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 809.6644164246909,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00010037499998816202,
+ "first_s": 0.0355080409999573,
+ "last_s": 2.522695915999975,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0004983329999959096,
+ "first_s": 0.12070795799996858,
+ "last_s": 2.52311737499997,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 29,
+ "accepted": 22,
+ "acceptance_rate": 0.7586206896551724,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0003972499999918,
+ "first_s": 0.05252608299997519,
+ "last_s": 2.522637457999963,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0003458749999936117,
+ "first_s": 0.06958604099997956,
+ "last_s": 2.5229750830000057,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.00017412500000091313,
+ "first_s": 0.018461415999979636,
+ "last_s": 2.5226144999999747,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0002841250000074069,
+ "first_s": 0.08661762499997394,
+ "last_s": 2.5230261250000012,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 28,
+ "accepted": 23,
+ "acceptance_rate": 0.8214285714285714,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.00023033299999042356,
+ "first_s": 0.1377665829999728,
+ "last_s": 2.5380237910000005,
+ "chunks": 238,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 237,
+ "drafted": 24,
+ "accepted": 18,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.080168776371308
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.1036599159999696,
+ "last_s": 2.523074832999953,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 27,
+ "accepted": 22,
+ "acceptance_rate": 0.8148148148148148,
+ "tokens_per_round": 1.0987124463519313
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 799.0692280251305,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.429200002841753e-05,
+ "first_s": 0.035924374999979136,
+ "last_s": 5.123591916999999,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00015887500001099397,
+ "first_s": 0.27468241700000817,
+ "last_s": 5.124402667000027,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0008406670000340455,
+ "first_s": 0.20647437500002752,
+ "last_s": 5.124251458999993,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002103749999946558,
+ "first_s": 0.087138374999995,
+ "last_s": 5.124144584000021,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0005356249999977081,
+ "first_s": 0.1893902090000097,
+ "last_s": 5.124441833999981,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002624169999876358,
+ "first_s": 0.25761429200002794,
+ "last_s": 5.124718541999982,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.000746375000005628,
+ "first_s": 0.07004283399999167,
+ "last_s": 5.124126208999996,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.000585166999997,
+ "first_s": 0.15526004200000898,
+ "last_s": 5.1245976249999785,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00031033400000524125,
+ "first_s": 0.22353983400000743,
+ "last_s": 5.124325959000032,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0007956670000339727,
+ "first_s": 0.052985625000019354,
+ "last_s": 5.124160416999985,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0006322090000026037,
+ "first_s": 0.17232549999999947,
+ "last_s": 5.12402079200001,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0003609999999980573,
+ "first_s": 0.01883658399998467,
+ "last_s": 5.123523834000025,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0006907920000003287,
+ "first_s": 0.24057662500001697,
+ "last_s": 5.124557625000023,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0004604169999993246,
+ "first_s": 0.12116666700001133,
+ "last_s": 5.124777166999991,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00040904200000113633,
+ "first_s": 0.1382068749999803,
+ "last_s": 5.124481709000008,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.10419675000002826,
+ "last_s": 5.123928625000019,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 359.5743061551645,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.02006162499998254,
+ "last_s": 0.7292335419999745,
+ "chunks": 91,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 90,
+ "drafted": 280,
+ "accepted": 165,
+ "acceptance_rate": 0.5892857142857143,
+ "tokens_per_round": 2.8444444444444446
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 617.636426500965,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.00010516700001517165,
+ "first_s": 0.04048800000003894,
+ "last_s": 1.6718915420000258,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00024808400002029884,
+ "first_s": 0.0793447500000184,
+ "last_s": 1.6719714590000194,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00018512500002998422,
+ "first_s": 0.0598595840000371,
+ "last_s": 1.6721314590000134,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.02067445900001985,
+ "last_s": 1.671916500000009,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 787.7309614581078,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.766700003410733e-05,
+ "first_s": 0.09887779199999613,
+ "last_s": 2.610538833000021,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 6,
+ "accepted": 2,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.00038387500001135777,
+ "first_s": 0.07951358299999356,
+ "last_s": 2.5942442080000205,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 8,
+ "accepted": 5,
+ "acceptance_rate": 0.625,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00016558300001179305,
+ "first_s": 0.02083362500002295,
+ "last_s": 2.5941928330000223,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 15,
+ "accepted": 5,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0005785420000279373,
+ "first_s": 0.15703295800000205,
+ "last_s": 2.6008843330000104,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 11,
+ "accepted": 4,
+ "acceptance_rate": 0.36363636363636365,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0002677919999882761,
+ "first_s": 0.13771400000001677,
+ "last_s": 2.6008761250000134,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 12,
+ "accepted": 4,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00021741699998756303,
+ "first_s": 0.040725208000026214,
+ "last_s": 2.594233250000002,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 9,
+ "accepted": 5,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0004375830000071801,
+ "first_s": 0.06012770800003864,
+ "last_s": 2.5942115000000285,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 8,
+ "accepted": 5,
+ "acceptance_rate": 0.625,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.11830204200003891,
+ "last_s": 2.6105502920000276,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 6,
+ "accepted": 2,
+ "acceptance_rate": 0.3333333333333333,
+ "tokens_per_round": 1.0118577075098814
+ }
+ }
+ ]
+ },
+ {
+ "round": 2,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 792.7530903708569,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.00010041699999874254,
+ "first_s": 0.23493779199998244,
+ "last_s": 5.167224250000004,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0005097499999919819,
+ "first_s": 0.021199499999966065,
+ "last_s": 5.166656958999965,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00016687499999079591,
+ "first_s": 0.3124321249999866,
+ "last_s": 5.167750834000003,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00044941699997025353,
+ "first_s": 0.04103566700001693,
+ "last_s": 5.167538583999999,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0005593340000018543,
+ "first_s": 0.21554237499998408,
+ "last_s": 5.167156292000016,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0002186670000128288,
+ "first_s": 0.11859137499999406,
+ "last_s": 5.167411125000001,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0007038749999992433,
+ "first_s": 0.15737291699997513,
+ "last_s": 5.167820875000018,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00026816700000154015,
+ "first_s": 0.06046645899999703,
+ "last_s": 5.166726417000007,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.000315875000012511,
+ "first_s": 0.17679329200001348,
+ "last_s": 5.167362916999991,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0008672089999777199,
+ "first_s": 0.25429754200001753,
+ "last_s": 5.167245292000018,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0007588749999740685,
+ "first_s": 0.2930327920000195,
+ "last_s": 5.167315083999995,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0003598750000151085,
+ "first_s": 0.13801645899997084,
+ "last_s": 5.167700583999988,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0006246250000003783,
+ "first_s": 0.2736417090000032,
+ "last_s": 5.167455874999973,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0008143750000044747,
+ "first_s": 0.19622391699999753,
+ "last_s": 5.1670614170000135,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00040524999997160194,
+ "first_s": 0.07984820899997658,
+ "last_s": 5.166772917000003,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.09918999999996458,
+ "last_s": 5.167650292000019,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.56191780544566,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.01761995799995475,
+ "last_s": 0.7347946249999495,
+ "chunks": 90,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 89,
+ "drafted": 301,
+ "accepted": 166,
+ "acceptance_rate": 0.5514950166112956,
+ "tokens_per_round": 2.8764044943820224
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 633.0814868973765,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.78340000301614e-05,
+ "first_s": 0.03571450000004006,
+ "last_s": 1.6046670000000063,
+ "chunks": 221,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 220,
+ "drafted": 54,
+ "accepted": 35,
+ "acceptance_rate": 0.6481481481481481,
+ "tokens_per_round": 1.1636363636363636
+ }
+ },
+ {
+ "sent_s": 0.0002466250000452419,
+ "first_s": 0.06973966700002165,
+ "last_s": 1.629312584000047,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 54,
+ "accepted": 33,
+ "acceptance_rate": 0.6111111111111112,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00017150000002175148,
+ "first_s": 0.018145584000023973,
+ "last_s": 1.6046562920000156,
+ "chunks": 221,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 220,
+ "drafted": 54,
+ "accepted": 35,
+ "acceptance_rate": 0.6481481481481481,
+ "tokens_per_round": 1.1636363636363636
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.05273579200002132,
+ "last_s": 1.629299250000031,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 55,
+ "accepted": 33,
+ "acceptance_rate": 0.6,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 813.2946971840239,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.820799999715746e-05,
+ "first_s": 0.018363375000035376,
+ "last_s": 2.491688291999992,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 42,
+ "accepted": 31,
+ "acceptance_rate": 0.7380952380952381,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.00041337499999372085,
+ "first_s": 0.13768166700003803,
+ "last_s": 2.5266792500000292,
+ "chunks": 230,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 229,
+ "drafted": 41,
+ "accepted": 26,
+ "acceptance_rate": 0.6341463414634146,
+ "tokens_per_round": 1.1179039301310043
+ }
+ },
+ {
+ "sent_s": 0.00016579200001842764,
+ "first_s": 0.05280904200003533,
+ "last_s": 2.4917732080000405,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 44,
+ "accepted": 31,
+ "acceptance_rate": 0.7045454545454546,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0005046250000191321,
+ "first_s": 0.1038404170000149,
+ "last_s": 2.4918392079999876,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 42,
+ "accepted": 30,
+ "acceptance_rate": 0.7142857142857143,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.00021883300001945827,
+ "first_s": 0.06984158300002719,
+ "last_s": 2.4920794999999885,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 44,
+ "accepted": 31,
+ "acceptance_rate": 0.7045454545454546,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0002707920000375452,
+ "first_s": 0.03581504200002428,
+ "last_s": 2.491924332999986,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 45,
+ "accepted": 30,
+ "acceptance_rate": 0.6666666666666666,
+ "tokens_per_round": 1.1377777777777778
+ }
+ },
+ {
+ "sent_s": 0.0003552919999947335,
+ "first_s": 0.1207515419999936,
+ "last_s": 2.4611360000000104,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 44,
+ "accepted": 32,
+ "acceptance_rate": 0.7272727272727273,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.08679779200002713,
+ "last_s": 2.4920344170000135,
+ "chunks": 226,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 225,
+ "drafted": 42,
+ "accepted": 30,
+ "acceptance_rate": 0.7142857142857143,
+ "tokens_per_round": 1.1377777777777778
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-drafted",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 886.8595214789708,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.116699999367484e-05,
+ "first_s": 0.03567537499998252,
+ "last_s": 4.528998417000025,
+ "chunks": 218,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 217,
+ "drafted": 59,
+ "accepted": 38,
+ "acceptance_rate": 0.6440677966101694,
+ "tokens_per_round": 1.1797235023041475
+ }
+ },
+ {
+ "sent_s": 0.0008545840000238059,
+ "first_s": 0.15475083400002632,
+ "last_s": 4.249944583999991,
+ "chunks": 203,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 202,
+ "drafted": 86,
+ "accepted": 53,
+ "acceptance_rate": 0.6162790697674418,
+ "tokens_per_round": 1.2673267326732673
+ }
+ },
+ {
+ "sent_s": 0.000156917000026624,
+ "first_s": 0.052642208999998275,
+ "last_s": 4.481628124999986,
+ "chunks": 215,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 214,
+ "drafted": 60,
+ "accepted": 41,
+ "acceptance_rate": 0.6833333333333333,
+ "tokens_per_round": 1.1962616822429906
+ }
+ },
+ {
+ "sent_s": 0.0005277920000139602,
+ "first_s": 0.2738892090000036,
+ "last_s": 4.615273375000015,
+ "chunks": 227,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 226,
+ "drafted": 40,
+ "accepted": 29,
+ "acceptance_rate": 0.725,
+ "tokens_per_round": 1.1327433628318584
+ }
+ },
+ {
+ "sent_s": 0.00045462500003168316,
+ "first_s": 0.2058160839999914,
+ "last_s": 4.5912452090000215,
+ "chunks": 223,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 222,
+ "drafted": 53,
+ "accepted": 33,
+ "acceptance_rate": 0.6226415094339622,
+ "tokens_per_round": 1.1531531531531531
+ }
+ },
+ {
+ "sent_s": 0.0002026669999963815,
+ "first_s": 0.171816917000001,
+ "last_s": 4.61525745900002,
+ "chunks": 227,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 226,
+ "drafted": 44,
+ "accepted": 29,
+ "acceptance_rate": 0.6590909090909091,
+ "tokens_per_round": 1.1327433628318584
+ }
+ },
+ {
+ "sent_s": 0.0010439999999789507,
+ "first_s": 0.25688379199999645,
+ "last_s": 4.481758458999991,
+ "chunks": 215,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 214,
+ "drafted": 53,
+ "accepted": 41,
+ "acceptance_rate": 0.7735849056603774,
+ "tokens_per_round": 1.1962616822429906
+ }
+ },
+ {
+ "sent_s": 0.0005842090000101052,
+ "first_s": 0.01870875000003025,
+ "last_s": 4.214412334000031,
+ "chunks": 201,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 200,
+ "drafted": 84,
+ "accepted": 55,
+ "acceptance_rate": 0.6547619047619048,
+ "tokens_per_round": 1.28
+ }
+ },
+ {
+ "sent_s": 0.00026120900002979397,
+ "first_s": 0.08659845900001528,
+ "last_s": 4.412412625000002,
+ "chunks": 211,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 210,
+ "drafted": 68,
+ "accepted": 45,
+ "acceptance_rate": 0.6617647058823529,
+ "tokens_per_round": 1.2190476190476192
+ }
+ },
+ {
+ "sent_s": 0.0003603339999926902,
+ "first_s": 0.1377227500000231,
+ "last_s": 4.619211875000019,
+ "chunks": 228,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 227,
+ "drafted": 47,
+ "accepted": 28,
+ "acceptance_rate": 0.5957446808510638,
+ "tokens_per_round": 1.1277533039647578
+ }
+ },
+ {
+ "sent_s": 0.00030870900002355484,
+ "first_s": 0.2228808749999871,
+ "last_s": 4.23052958400001,
+ "chunks": 202,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 201,
+ "drafted": 68,
+ "accepted": 54,
+ "acceptance_rate": 0.7941176470588235,
+ "tokens_per_round": 1.2736318407960199
+ }
+ },
+ {
+ "sent_s": 0.0007665840000186108,
+ "first_s": 0.23986387500002593,
+ "last_s": 4.0215202499999805,
+ "chunks": 191,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 190,
+ "drafted": 83,
+ "accepted": 66,
+ "acceptance_rate": 0.7951807228915663,
+ "tokens_per_round": 1.3473684210526315
+ }
+ },
+ {
+ "sent_s": 0.0006267090000164899,
+ "first_s": 0.069626583999991,
+ "last_s": 4.5292172919999985,
+ "chunks": 218,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 217,
+ "drafted": 51,
+ "accepted": 38,
+ "acceptance_rate": 0.7450980392156863,
+ "tokens_per_round": 1.1797235023041475
+ }
+ },
+ {
+ "sent_s": 0.0004082920000314516,
+ "first_s": 0.10361899999998059,
+ "last_s": 4.591206166999996,
+ "chunks": 223,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 222,
+ "drafted": 48,
+ "accepted": 33,
+ "acceptance_rate": 0.6875,
+ "tokens_per_round": 1.1531531531531531
+ }
+ },
+ {
+ "sent_s": 0.0006866250000143737,
+ "first_s": 0.18879587499998252,
+ "last_s": 4.5630358750000255,
+ "chunks": 220,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 219,
+ "drafted": 51,
+ "accepted": 36,
+ "acceptance_rate": 0.7058823529411765,
+ "tokens_per_round": 1.1689497716894977
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.12069558400003189,
+ "last_s": 4.514597999999978,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 60,
+ "accepted": 39,
+ "acceptance_rate": 0.65,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.75662063341406,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020067166999979236,
+ "last_s": 0.7348401669999589,
+ "chunks": 92,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 615.2096412258015,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.00010383299996874484,
+ "first_s": 0.04034483300000602,
+ "last_s": 1.678376582999988,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0002540830000157257,
+ "first_s": 0.07913179199999831,
+ "last_s": 1.6784027079999646,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00017783300000928648,
+ "first_s": 0.059760707999998885,
+ "last_s": 1.6784122079999975,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.02044079199998805,
+ "last_s": 1.6783869170000116,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.9578088973333,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.466599999541359e-05,
+ "first_s": 0.0795916659999989,
+ "last_s": 2.580623707999962,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 5,
+ "accepted": 2,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0118577075098814
+ }
+ },
+ {
+ "sent_s": 0.00016291599996520745,
+ "first_s": 0.02094016600000259,
+ "last_s": 2.5663399579999577,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0003404159999718104,
+ "first_s": 0.06019687499997417,
+ "last_s": 2.566447832999984,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0005412079999587149,
+ "first_s": 0.13779554100000269,
+ "last_s": 2.566840791000004,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 16,
+ "accepted": 5,
+ "acceptance_rate": 0.3125,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00021995800000240706,
+ "first_s": 0.040804041000001234,
+ "last_s": 2.5664145000000076,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0004180409999889889,
+ "first_s": 0.15715837499999452,
+ "last_s": 2.566752249999979,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 16,
+ "accepted": 5,
+ "acceptance_rate": 0.3125,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.00027199999999538704,
+ "first_s": 0.11837312500000507,
+ "last_s": 2.5667976659999567,
+ "chunks": 251,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 250,
+ "drafted": 9,
+ "accepted": 5,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.024
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.09895945799996753,
+ "last_s": 2.5806741659999943,
+ "chunks": 254,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 253,
+ "drafted": 5,
+ "accepted": 2,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0118577075098814
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-drafted",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 854.4605435798837,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 8.824999997614213e-05,
+ "first_s": 0.02114387500000703,
+ "last_s": 4.769526541999994,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 16,
+ "accepted": 6,
+ "acceptance_rate": 0.375,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00040862499997729174,
+ "first_s": 0.09924091699997462,
+ "last_s": 4.016953583999964,
+ "chunks": 196,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 195,
+ "drafted": 72,
+ "accepted": 60,
+ "acceptance_rate": 0.8333333333333334,
+ "tokens_per_round": 1.3128205128205128
+ }
+ },
+ {
+ "sent_s": 0.0009741669999812075,
+ "first_s": 0.15741904199995815,
+ "last_s": 4.749796083999968,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 15,
+ "accepted": 7,
+ "acceptance_rate": 0.4666666666666667,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00015629199998556942,
+ "first_s": 0.312493083999982,
+ "last_s": 4.795797708999999,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 5,
+ "acceptance_rate": 0.5555555555555556,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005290419999823825,
+ "first_s": 0.2930928339999923,
+ "last_s": 4.796028833999969,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0002106249999656029,
+ "first_s": 0.21556441699999596,
+ "last_s": 4.79570874999996,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 12,
+ "accepted": 5,
+ "acceptance_rate": 0.4166666666666667,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0003591249999885804,
+ "first_s": 0.1186404999999695,
+ "last_s": 4.3597545839999725,
+ "chunks": 219,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 58,
+ "accepted": 37,
+ "acceptance_rate": 0.6379310344827587,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.0008440839999934724,
+ "first_s": 0.13799995899995565,
+ "last_s": 4.562454749999972,
+ "chunks": 234,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 34,
+ "accepted": 22,
+ "acceptance_rate": 0.6470588235294118,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0006364999999846077,
+ "first_s": 0.27367270899998175,
+ "last_s": 4.7960864169999695,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 10,
+ "accepted": 5,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0002579169999989972,
+ "first_s": 0.25427770900000723,
+ "last_s": 4.795861041999956,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 11,
+ "accepted": 5,
+ "acceptance_rate": 0.45454545454545453,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006784589999710988,
+ "first_s": 0.23492499999997563,
+ "last_s": 4.795682749999969,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 11,
+ "accepted": 5,
+ "acceptance_rate": 0.45454545454545453,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00030837499997460327,
+ "first_s": 0.17680974999996124,
+ "last_s": 4.648196208999991,
+ "chunks": 241,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 240,
+ "drafted": 29,
+ "accepted": 15,
+ "acceptance_rate": 0.5172413793103449,
+ "tokens_per_round": 1.0666666666666667
+ }
+ },
+ {
+ "sent_s": 0.0005799999999567262,
+ "first_s": 0.06045579199997064,
+ "last_s": 3.590620458999979,
+ "chunks": 169,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 168,
+ "drafted": 123,
+ "accepted": 87,
+ "acceptance_rate": 0.7073170731707317,
+ "tokens_per_round": 1.5238095238095237
+ }
+ },
+ {
+ "sent_s": 0.0007273749999967549,
+ "first_s": 0.04103145900000982,
+ "last_s": 4.769692292000002,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 15,
+ "accepted": 6,
+ "acceptance_rate": 0.4,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.000459708999983377,
+ "first_s": 0.07988737499999843,
+ "last_s": 4.402278833999958,
+ "chunks": 222,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 221,
+ "drafted": 51,
+ "accepted": 34,
+ "acceptance_rate": 0.6666666666666666,
+ "tokens_per_round": 1.158371040723982
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.19619170900000427,
+ "last_s": 4.795631083999979,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 14,
+ "accepted": 5,
+ "acceptance_rate": 0.35714285714285715,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 288.0685129679525,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019108499999958894,
+ "last_s": 0.904314582999973,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 652.4009218675697,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00010487499997680061,
+ "first_s": 0.0179946669999822,
+ "last_s": 1.5776035000000093,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00017887500001734224,
+ "first_s": 0.06711383399999704,
+ "last_s": 1.5814504589999956,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 46,
+ "accepted": 31,
+ "acceptance_rate": 0.6739130434782609,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0002606250000098953,
+ "first_s": 0.03437545899998895,
+ "last_s": 1.5775790000000143,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.050728042000002915,
+ "last_s": 1.5775919999999815,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 831.6126825307226,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.912500001973967e-05,
+ "first_s": 0.025430374999984906,
+ "last_s": 2.47072754200002,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00045554199999742195,
+ "first_s": 0.09088358399998242,
+ "last_s": 2.471106458999998,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00035112500000877844,
+ "first_s": 0.1072000839999987,
+ "last_s": 2.4711619999999925,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0002247500000294167,
+ "first_s": 0.07449875000003203,
+ "last_s": 2.471292917000028,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00017095900000185793,
+ "first_s": 0.13993212500002983,
+ "last_s": 2.4784954170000333,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.0005192919999785772,
+ "first_s": 0.12356725000000779,
+ "last_s": 2.471208625000031,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0002811250000149812,
+ "first_s": 0.041804958999989594,
+ "last_s": 2.470745749999992,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.058155791999979556,
+ "last_s": 2.4708504170000083,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "main",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 830.0623561457346,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.433299999273004e-05,
+ "first_s": 0.05835183300001745,
+ "last_s": 4.939848499999982,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.0008314579999932903,
+ "first_s": 0.1728316250000148,
+ "last_s": 4.940065875000016,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.0007320000000277105,
+ "first_s": 0.12382362499999999,
+ "last_s": 4.940850375000025,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.0012377919999835285,
+ "first_s": 0.2546014999999784,
+ "last_s": 4.940907750000008,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.00016191700001400022,
+ "first_s": 0.02561437500003194,
+ "last_s": 4.939799875000006,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.00042754200001127174,
+ "first_s": 0.0911155000000008,
+ "last_s": 4.94076445799999,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.00021383300003208205,
+ "first_s": 0.22193470799999204,
+ "last_s": 4.940359833000002,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.00026975000002948946,
+ "first_s": 0.1074880830000211,
+ "last_s": 4.940171292000002,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.0009474999999952161,
+ "first_s": 0.1892031670000165,
+ "last_s": 4.9407295419999855,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.0007826670000099512,
+ "first_s": 0.07470058299998072,
+ "last_s": 4.939863457999991,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.0005763330000263522,
+ "first_s": 0.20557658299998138,
+ "last_s": 4.940600042000028,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.00047649999999066495,
+ "first_s": 0.04199012499998389,
+ "last_s": 4.9398321250000095,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.00032729200000858327,
+ "first_s": 0.15649383300001318,
+ "last_s": 4.940804249999985,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.00037887500002398156,
+ "first_s": 0.1401767080000127,
+ "last_s": 4.940281874999982,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.0010184999999864885,
+ "first_s": 0.2709963330000278,
+ "last_s": 4.940641292000009,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.2382644580000033,
+ "last_s": 4.940421125,
+ "chunks": 239,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 238,
+ "drafted": 20,
+ "accepted": 17,
+ "acceptance_rate": 0.85,
+ "tokens_per_round": 1.0756302521008403
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 261.7083370911104,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020360124999967866,
+ "last_s": 0.9947272499999826,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 622.3375716098092,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.00010224999999763895,
+ "first_s": 0.027587374999995973,
+ "last_s": 1.666514415999984,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00025445799997214635,
+ "first_s": 0.08412108300001364,
+ "last_s": 1.666569250000009,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0001738330000193855,
+ "first_s": 0.06530154099999663,
+ "last_s": 1.666530458000011,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.04645841600000722,
+ "last_s": 1.66653974999997,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 804.5208417575939,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.104200000820128e-05,
+ "first_s": 0.020772958000009112,
+ "last_s": 2.5524102919999905,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00031758300002593387,
+ "first_s": 0.11569991700002902,
+ "last_s": 2.5526911670000345,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00021054199999070988,
+ "first_s": 0.1345285419999982,
+ "last_s": 2.5528591249999977,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00015941700002031212,
+ "first_s": 0.04024687500003665,
+ "last_s": 2.5524276670000177,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00026524999998400745,
+ "first_s": 0.09682808299999124,
+ "last_s": 2.5529229580000106,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00046179200001006393,
+ "first_s": 0.07796595799999295,
+ "last_s": 2.5530045830000176,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00039845800000648524,
+ "first_s": 0.05911545800000795,
+ "last_s": 2.552473833000022,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.1533286669999825,
+ "last_s": 2.556443750000028,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "main",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 797.6543488046647,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.82920000183185e-05,
+ "first_s": 0.05991379200003166,
+ "last_s": 5.135538708000013,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007642920000421327,
+ "first_s": 0.28572975000003,
+ "last_s": 5.136203917000046,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00032704200003763617,
+ "first_s": 0.0975009170000476,
+ "last_s": 5.135654750000015,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005441250000330911,
+ "first_s": 0.1539891250000096,
+ "last_s": 5.13605429200004,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00017058299999916926,
+ "first_s": 0.11633641700001363,
+ "last_s": 5.136342542000023,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0009615000000167129,
+ "first_s": 0.17281345800000736,
+ "last_s": 5.135958791999997,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00022329200004378436,
+ "first_s": 0.21046737500000745,
+ "last_s": 5.136289792000014,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00042820800001663883,
+ "first_s": 0.22928483300000835,
+ "last_s": 5.136428667000018,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0002809169999977712,
+ "first_s": 0.1351456670000175,
+ "last_s": 5.136243083000011,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00047337500001276567,
+ "first_s": 0.26692425000004505,
+ "last_s": 5.135901042,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006765830000290407,
+ "first_s": 0.3047265420000258,
+ "last_s": 5.136158792000003,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005874580000408969,
+ "first_s": 0.2481052919999911,
+ "last_s": 5.1363874580000015,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00037908300004119155,
+ "first_s": 0.041061291999994864,
+ "last_s": 5.135998167000025,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0006305420000103368,
+ "first_s": 0.0787272919999964,
+ "last_s": 5.135376792000045,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007224999999948523,
+ "first_s": 0.1916815000000156,
+ "last_s": 5.1358073330000025,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.021431167000002915,
+ "last_s": 5.13525258300001,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 355.3981583058367,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019139250000023367,
+ "last_s": 0.7366443750000258,
+ "chunks": 90,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 89,
+ "drafted": 301,
+ "accepted": 166,
+ "acceptance_rate": 0.5514950166112956,
+ "tokens_per_round": 2.8764044943820224
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 645.6944342132804,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0001023330000293754,
+ "first_s": 0.07652729200003705,
+ "last_s": 1.595149417000016,
+ "chunks": 223,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 222,
+ "drafted": 41,
+ "accepted": 33,
+ "acceptance_rate": 0.8048780487804879,
+ "tokens_per_round": 1.1531531531531531
+ }
+ },
+ {
+ "sent_s": 0.00023770800004285775,
+ "first_s": 0.025366125000005013,
+ "last_s": 1.6011396250000303,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00017637500002365414,
+ "first_s": 0.04245066700002553,
+ "last_s": 1.6011511250000012,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 41,
+ "accepted": 32,
+ "acceptance_rate": 0.7804878048780488,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.05945966700005556,
+ "last_s": 1.6050607080000532,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 817.5820896934242,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.62500000127875e-05,
+ "first_s": 0.03566116700000066,
+ "last_s": 2.4972394590000135,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0002860840000380449,
+ "first_s": 0.018587209000031635,
+ "last_s": 2.497224417000041,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0004294590000313292,
+ "first_s": 0.12091908400003604,
+ "last_s": 2.5022188340000184,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00017379199999822958,
+ "first_s": 0.05266087500001504,
+ "last_s": 2.497260959000016,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.00022987500000226646,
+ "first_s": 0.13794095899999093,
+ "last_s": 2.513749500000017,
+ "chunks": 238,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 237,
+ "drafted": 25,
+ "accepted": 18,
+ "acceptance_rate": 0.72,
+ "tokens_per_round": 1.080168776371308
+ }
+ },
+ {
+ "sent_s": 0.0005138750000241998,
+ "first_s": 0.10387541700004022,
+ "last_s": 2.4977113749999944,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.000378334000004088,
+ "first_s": 0.08678854200002206,
+ "last_s": 2.4975427090000153,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.06975508400000763,
+ "last_s": 2.4973653750000153,
+ "chunks": 234,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 233,
+ "drafted": 26,
+ "accepted": 22,
+ "acceptance_rate": 0.8461538461538461,
+ "tokens_per_round": 1.0987124463519313
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 798.2396077591403,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.858300001042153e-05,
+ "first_s": 0.17979529199999433,
+ "last_s": 5.137077792000014,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0007607500000403888,
+ "first_s": 0.26507354200003874,
+ "last_s": 5.1373404170000185,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0001688749999857464,
+ "first_s": 0.14569525000001704,
+ "last_s": 5.13666766700004,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0005366249999951833,
+ "first_s": 0.09445470800000066,
+ "last_s": 5.137228125000036,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00022016700000904166,
+ "first_s": 0.24801995799998622,
+ "last_s": 5.13726516700001,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0005775000000198816,
+ "first_s": 0.16273537500001112,
+ "last_s": 5.136763833000032,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0006260420000216982,
+ "first_s": 0.06030212500002108,
+ "last_s": 5.136775083000032,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0002752920000261838,
+ "first_s": 0.11154883300002894,
+ "last_s": 5.137179000000003,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00047579200003156075,
+ "first_s": 0.04323062500003516,
+ "last_s": 5.136403292000011,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00032704200003763617,
+ "first_s": 0.026093166999999085,
+ "last_s": 5.136304792000033,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0008387080000034075,
+ "first_s": 0.2821312500000204,
+ "last_s": 5.137132000000008,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0003770000000145046,
+ "first_s": 0.1968186249999917,
+ "last_s": 5.13730370799999,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0008963750000248183,
+ "first_s": 0.0773855420000018,
+ "last_s": 5.136950667000008,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0006831670000337908,
+ "first_s": 0.2138946250000231,
+ "last_s": 5.1368389579999985,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.00042737500001521767,
+ "first_s": 0.12860412500003804,
+ "last_s": 5.136904917000038,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.23097549999999956,
+ "last_s": 5.137033458000019,
+ "chunks": 253,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 4,
+ "accepted": 3,
+ "acceptance_rate": 0.75,
+ "tokens_per_round": 1.0158730158730158
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 356.16467232003214,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.020453832999976385,
+ "last_s": 0.7364147909999588,
+ "chunks": 92,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 91,
+ "drafted": 283,
+ "accepted": 164,
+ "acceptance_rate": 0.5795053003533569,
+ "tokens_per_round": 2.8131868131868134
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 618.3910873475105,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.00010041600000931794,
+ "first_s": 0.027887832999965667,
+ "last_s": 1.6771637909999981,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00017733299995370544,
+ "first_s": 0.04744333299998971,
+ "last_s": 1.6771732079999992,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.00023887499997954365,
+ "first_s": 0.08631983300000456,
+ "last_s": 1.6773294579999742,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 17,
+ "accepted": 6,
+ "acceptance_rate": 0.35294117647058826,
+ "tokens_per_round": 1.0281124497991967
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.06689787499999511,
+ "last_s": 1.6771853329999544,
+ "chunks": 250,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 249,
+ "drafted": 11,
+ "accepted": 6,
+ "acceptance_rate": 0.5454545454545454,
+ "tokens_per_round": 1.0281124497991967
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 796.428603323352,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.195899997394008e-05,
+ "first_s": 0.0462688339999886,
+ "last_s": 2.5843012919999637,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.000409708999995928,
+ "first_s": 0.16280487500000618,
+ "last_s": 2.5882531249999943,
+ "chunks": 256,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 255,
+ "drafted": 6,
+ "accepted": 0,
+ "acceptance_rate": 0.0,
+ "tokens_per_round": 1.003921568627451
+ }
+ },
+ {
+ "sent_s": 0.00016158400001131668,
+ "first_s": 0.10456841699999586,
+ "last_s": 2.5847895839999637,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0002703749999568572,
+ "first_s": 0.06574124999997366,
+ "last_s": 2.5843564999999558,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00021745899999814355,
+ "first_s": 0.08518424999999752,
+ "last_s": 2.584339166999996,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0005527089999759482,
+ "first_s": 0.14339274999997542,
+ "last_s": 2.584606416999975,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00034283399998003006,
+ "first_s": 0.12398116700001083,
+ "last_s": 2.584396708999975,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.026818249999962518,
+ "last_s": 2.5842485839999654,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 4,
+ "accepted": 1,
+ "acceptance_rate": 0.25,
+ "tokens_per_round": 1.0078740157480315
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "published",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 791.8338766948201,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.87500000064756e-05,
+ "first_s": 0.22255808299996716,
+ "last_s": 5.180806458000006,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0007360000000176115,
+ "first_s": 0.18369995799997696,
+ "last_s": 5.180368541000007,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00022083300001440875,
+ "first_s": 0.12557158300000992,
+ "last_s": 5.180714582999997,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0006382909999729236,
+ "first_s": 0.2806921249999732,
+ "last_s": 5.180587749999972,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00016958300000169402,
+ "first_s": 0.028537165999978242,
+ "last_s": 5.180937416000006,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0008457910000174707,
+ "first_s": 0.14491249999997535,
+ "last_s": 5.180527457999972,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0005273329999795351,
+ "first_s": 0.0479936250000037,
+ "last_s": 5.179904207999982,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00027062499998464773,
+ "first_s": 0.20311637499997914,
+ "last_s": 5.180457082999965,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0003679579999698035,
+ "first_s": 0.16430674999998018,
+ "last_s": 5.180646958000011,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0005860409999627336,
+ "first_s": 0.10620045799998934,
+ "last_s": 5.179984499999989,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0007962909999719159,
+ "first_s": 0.3195373749999817,
+ "last_s": 5.181050749999997,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0006836249999651045,
+ "first_s": 0.30011095800000476,
+ "last_s": 5.181133082999963,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0004127500000095097,
+ "first_s": 0.261269249999998,
+ "last_s": 5.180873165999969,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00032120800000257077,
+ "first_s": 0.0674458329999652,
+ "last_s": 5.180086749999987,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.00045554100000799735,
+ "first_s": 0.24194262499997876,
+ "last_s": 5.181006707999984,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.08683024999999134,
+ "last_s": 5.180760291000013,
+ "chunks": 255,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 254,
+ "drafted": 2,
+ "accepted": 1,
+ "acceptance_rate": 0.5,
+ "tokens_per_round": 1.0078740157480315
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 1,
+ "tps": 289.11549612823666,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019241749999991953,
+ "last_s": 0.9012422079999851,
+ "chunks": 217,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 216,
+ "drafted": 100,
+ "accepted": 39,
+ "acceptance_rate": 0.39,
+ "tokens_per_round": 1.1851851851851851
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 4,
+ "tps": 661.6906341663944,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 0.00010058300000537201,
+ "first_s": 0.018040291999966485,
+ "last_s": 1.5595010829999865,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 63,
+ "accepted": 37,
+ "acceptance_rate": 0.5873015873015873,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.00024054199997181058,
+ "first_s": 0.03435920800001213,
+ "last_s": 1.5595186670000203,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 63,
+ "accepted": 37,
+ "acceptance_rate": 0.5873015873015873,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.00018033300000297459,
+ "first_s": 0.06704200000001492,
+ "last_s": 1.5595461670000077,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.05069608299999118,
+ "last_s": 1.5595314999999914,
+ "chunks": 219,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 218,
+ "drafted": 54,
+ "accepted": 37,
+ "acceptance_rate": 0.6851851851851852,
+ "tokens_per_round": 1.1743119266055047
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 8,
+ "tps": 851.92509905097,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.675000001152512e-05,
+ "first_s": 0.12320379200002662,
+ "last_s": 2.4198644169999852,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.0005418329999997695,
+ "first_s": 0.0742406670000264,
+ "last_s": 2.41535650000003,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00017433300001812313,
+ "first_s": 0.13952354200000627,
+ "last_s": 2.419875917000013,
+ "chunks": 225,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 224,
+ "drafted": 40,
+ "accepted": 31,
+ "acceptance_rate": 0.775,
+ "tokens_per_round": 1.1428571428571428
+ }
+ },
+ {
+ "sent_s": 0.00036108300002979377,
+ "first_s": 0.10689100000001872,
+ "last_s": 2.415414583000029,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00027833299998292205,
+ "first_s": 0.025299207999978535,
+ "last_s": 2.414964291999979,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0004121250000252985,
+ "first_s": 0.09056883300002028,
+ "last_s": 2.4152982080000243,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 42,
+ "accepted": 32,
+ "acceptance_rate": 0.7619047619047619,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.00022708300002705073,
+ "first_s": 0.04163458300001821,
+ "last_s": 2.4150827500000105,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.057957625000028656,
+ "last_s": 2.4149834579999947,
+ "chunks": 224,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 223,
+ "drafted": 43,
+ "accepted": 32,
+ "acceptance_rate": 0.7441860465116279,
+ "tokens_per_round": 1.147982062780269
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-plain",
+ "prompt": "code",
+ "streams": 16,
+ "tps": 849.7539917935715,
+ "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc",
+ "requests": [
+ {
+ "sent_s": 9.229200003346705e-05,
+ "first_s": 0.1563097500000481,
+ "last_s": 4.817629250000039,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0007655420000105551,
+ "first_s": 0.09106545800000276,
+ "last_s": 4.818126333000009,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0005632080000168571,
+ "first_s": 0.025629167000033704,
+ "last_s": 4.817198333000022,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00021450000002687375,
+ "first_s": 0.20527954200002796,
+ "last_s": 4.817866583000011,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0001658750000501641,
+ "first_s": 0.05837937500001544,
+ "last_s": 4.817788542000017,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0009786670000266895,
+ "first_s": 0.2706689580000443,
+ "last_s": 4.827018792000047,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 42,
+ "accepted": 19,
+ "acceptance_rate": 0.4523809523809524,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.0006769170000211489,
+ "first_s": 0.12370716700002049,
+ "last_s": 4.81748595800002,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00027220800001259704,
+ "first_s": 0.22165316699999948,
+ "last_s": 4.817694750000044,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0003266670000243721,
+ "first_s": 0.2543046670000422,
+ "last_s": 4.827006958000027,
+ "chunks": 237,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 236,
+ "drafted": 49,
+ "accepted": 19,
+ "acceptance_rate": 0.3877551020408163,
+ "tokens_per_round": 1.0847457627118644
+ }
+ },
+ {
+ "sent_s": 0.0007246250000321197,
+ "first_s": 0.1889802920000534,
+ "last_s": 4.818011333000015,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0006133750000003602,
+ "first_s": 0.04200358300005291,
+ "last_s": 4.817208667000045,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00037904200002003563,
+ "first_s": 0.13998337500004254,
+ "last_s": 4.817958458000021,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.00047962500002540764,
+ "first_s": 0.0747155829999997,
+ "last_s": 4.818229042000041,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0011613750000378786,
+ "first_s": 0.2379564170000208,
+ "last_s": 4.818185667000023,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0004290420000074846,
+ "first_s": 0.17260258300001396,
+ "last_s": 4.817737625000007,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.10742200000004232,
+ "last_s": 4.817548625000029,
+ "chunks": 235,
+ "prompt_tokens": 14,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 234,
+ "drafted": 25,
+ "accepted": 21,
+ "acceptance_rate": 0.84,
+ "tokens_per_round": 1.0940170940170941
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 1,
+ "tps": 261.22060964024485,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 0.0,
+ "first_s": 0.019794375000003583,
+ "last_s": 0.9959807500000011,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 72,
+ "accepted": 8,
+ "acceptance_rate": 0.1111111111111111,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 4,
+ "tps": 609.6870758668199,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.979099996826335e-05,
+ "first_s": 0.027631165999991936,
+ "last_s": 1.7005795000000035,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 35,
+ "accepted": 8,
+ "acceptance_rate": 0.22857142857142856,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.0002387909999583826,
+ "first_s": 0.08411141599998473,
+ "last_s": 1.700620540999978,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.00017691599998670426,
+ "first_s": 0.046487874999968426,
+ "last_s": 1.7005932499999972,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 35,
+ "accepted": 8,
+ "acceptance_rate": 0.22857142857142856,
+ "tokens_per_round": 1.0364372469635628
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.06533966599999985,
+ "last_s": 1.7006090409999501,
+ "chunks": 248,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 247,
+ "drafted": 30,
+ "accepted": 8,
+ "acceptance_rate": 0.26666666666666666,
+ "tokens_per_round": 1.0364372469635628
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 8,
+ "tps": 809.9225170234652,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.008300003188197e-05,
+ "first_s": 0.02065962500000751,
+ "last_s": 2.538832582999987,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00031466600000840117,
+ "first_s": 0.0401318330000322,
+ "last_s": 2.538852875000032,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00016016600000057224,
+ "first_s": 0.05894499999999425,
+ "last_s": 2.5388885830000163,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00037554099998260426,
+ "first_s": 0.07783116600000994,
+ "last_s": 2.539023708000002,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0004380000000310247,
+ "first_s": 0.11547354100002849,
+ "last_s": 2.5393518750000226,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.00020958300001439056,
+ "first_s": 0.1531026250000309,
+ "last_s": 2.539231583000003,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0002641660000222146,
+ "first_s": 0.09667970800001058,
+ "last_s": 2.538900791000003,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 18,
+ "accepted": 7,
+ "acceptance_rate": 0.3888888888888889,
+ "tokens_per_round": 1.032258064516129
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.13430062500003714,
+ "last_s": 2.5394190829999843,
+ "chunks": 249,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 248,
+ "drafted": 12,
+ "accepted": 7,
+ "acceptance_rate": 0.5833333333333334,
+ "tokens_per_round": 1.032258064516129
+ }
+ }
+ ]
+ },
+ {
+ "round": 3,
+ "arm": "candidate-plain",
+ "prompt": "prose",
+ "streams": 16,
+ "tps": 802.5799140637832,
+ "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c",
+ "requests": [
+ {
+ "sent_s": 9.554199999683988e-05,
+ "first_s": 0.10360991700002842,
+ "last_s": 5.107608249999998,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00016604199998937474,
+ "first_s": 0.14120920800002068,
+ "last_s": 5.107471332999978,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005204170000183694,
+ "first_s": 0.17878975000002129,
+ "last_s": 5.107176707999997,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007186670000010054,
+ "first_s": 0.21637195800002473,
+ "last_s": 5.107288041999993,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00021729200000208948,
+ "first_s": 0.12241758300001493,
+ "last_s": 5.107812749999994,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00032437499999105057,
+ "first_s": 0.25404087499998695,
+ "last_s": 5.10714120800003,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.00027066699999522825,
+ "first_s": 0.028251749999981257,
+ "last_s": 5.106763291999982,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0004718750000165528,
+ "first_s": 0.27287325000003193,
+ "last_s": 5.107762917000002,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0008905419999791775,
+ "first_s": 0.2352070829999775,
+ "last_s": 5.107568833000016,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0007765419999827827,
+ "first_s": 0.19758012500000177,
+ "last_s": 5.107524416999979,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0005857080000168935,
+ "first_s": 0.31045379200003254,
+ "last_s": 5.111857666999981,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 30,
+ "accepted": 3,
+ "acceptance_rate": 0.1,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0003726670000219201,
+ "first_s": 0.06591500000001815,
+ "last_s": 5.106777333000025,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.000637874999995347,
+ "first_s": 0.08476883299999827,
+ "last_s": 5.107337874999985,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0004238750000240543,
+ "first_s": 0.16000133300002517,
+ "last_s": 5.107853707999993,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ },
+ {
+ "sent_s": 0.0009781250000173713,
+ "first_s": 0.29165512499997703,
+ "last_s": 5.111846125,
+ "chunks": 253,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 252,
+ "drafted": 35,
+ "accepted": 3,
+ "acceptance_rate": 0.08571428571428572,
+ "tokens_per_round": 1.0158730158730158
+ }
+ },
+ {
+ "sent_s": 0.0,
+ "first_s": 0.04710833299998285,
+ "last_s": 5.106851082999981,
+ "chunks": 252,
+ "prompt_tokens": 18,
+ "completion_tokens": 256,
+ "speculative": {
+ "rounds": 251,
+ "drafted": 9,
+ "accepted": 4,
+ "acceptance_rate": 0.4444444444444444,
+ "tokens_per_round": 1.0199203187250996
+ }
+ }
+ ]
+ }
+ ],
+ "medians": [
+ {
+ "prompt": "code",
+ "streams": 1,
+ "tps": {
+ "main": 289.3127543896282,
+ "published": 354.63683817375284,
+ "candidate-plain": 288.9588092703433,
+ "candidate-drafted": 356.3379766224038
+ },
+ "candidate_drafted_vs_published_pct": 0.47968464229808383,
+ "candidate_drafted_vs_main_pct": 23.167047154274535,
+ "candidate_plain_vs_main_pct": -0.12233996390226265
+ },
+ {
+ "prompt": "code",
+ "streams": 4,
+ "tps": {
+ "main": 652.2735791542232,
+ "published": 645.4865332378143,
+ "candidate-plain": 661.7877417761589,
+ "candidate-drafted": 633.0452266294983
+ },
+ "candidate_drafted_vs_published_pct": -1.9274308552820418,
+ "candidate_drafted_vs_main_pct": -2.94789688548438,
+ "candidate_plain_vs_main_pct": 1.4586153610993025
+ },
+ {
+ "prompt": "code",
+ "streams": 8,
+ "tps": {
+ "main": 830.8723123556888,
+ "published": 812.6499883768286,
+ "candidate-plain": 852.0692879199623,
+ "candidate-drafted": 812.5992670903888
+ },
+ "candidate_drafted_vs_published_pct": -0.00624146768783973,
+ "candidate_drafted_vs_main_pct": -2.199260342842846,
+ "candidate_plain_vs_main_pct": 2.5511712508719864
+ },
+ {
+ "prompt": "code",
+ "streams": 16,
+ "tps": {
+ "main": 830.128290576471,
+ "published": 798.8767760220219,
+ "candidate-plain": 849.7420828208678,
+ "candidate-drafted": 886.4564675049711
+ },
+ "candidate_drafted_vs_published_pct": 10.962853610421508,
+ "candidate_drafted_vs_main_pct": 6.785478529997291,
+ "candidate_plain_vs_main_pct": 2.3627422974316747
+ },
+ {
+ "prompt": "prose",
+ "streams": 1,
+ "tps": {
+ "main": 262.30748385704317,
+ "published": 356.77452074457415,
+ "candidate-plain": 261.3022597821349,
+ "candidate-drafted": 356.9475919964822
+ },
+ "candidate_drafted_vs_published_pct": 0.04850998091086023,
+ "candidate_drafted_vs_main_pct": 36.07983529399321,
+ "candidate_plain_vs_main_pct": -0.38322355890391124
+ },
+ {
+ "prompt": "prose",
+ "streams": 4,
+ "tps": {
+ "main": 622.7876024155925,
+ "published": 618.2932356793516,
+ "candidate-plain": 610.0894226933044,
+ "candidate-drafted": 615.7268551631641
+ },
+ "candidate_drafted_vs_published_pct": -0.41507497868186194,
+ "candidate_drafted_vs_main_pct": -1.1337327886813964,
+ "candidate_plain_vs_main_pct": -2.038926220277326
+ },
+ {
+ "prompt": "prose",
+ "streams": 8,
+ "tps": {
+ "main": 804.4487723340825,
+ "published": 792.2780785659455,
+ "candidate-plain": 810.0205684276797,
+ "candidate-drafted": 796.214425867501
+ },
+ "candidate_drafted_vs_published_pct": 0.49683910334619164,
+ "candidate_drafted_vs_main_pct": -1.0236010980152144,
+ "candidate_plain_vs_main_pct": 0.6926228599281492
+ },
+ {
+ "prompt": "prose",
+ "streams": 16,
+ "tps": {
+ "main": 798.6060418170423,
+ "published": 792.6643643406294,
+ "candidate-plain": 802.7420492371732,
+ "candidate-drafted": 862.9112022985937
+ },
+ "candidate_drafted_vs_published_pct": 8.862116315320744,
+ "candidate_drafted_vs_main_pct": 8.052175555201146,
+ "candidate_plain_vs_main_pct": 0.5179033470270822
+ }
+ ],
+ "m5_qualification": {
+ "operator_summary": "SUMMARY 11 kernels: 11 pass gates 1-4, 0 fail, 0 known findings; 17 controls: 17 red as planted, 0 missed; 0 limit problems; gates 53558 ms, limits 110 ms",
+ "drafter_reference": {
+ "picks_equal": 9,
+ "picks": 9,
+ "worst_confidence_difference": 0.00158,
+ "steady_rounds": 20,
+ "host_allocations": 0,
+ "resizes": 0
+ },
+ "stress": [
+ {
+ "slots": 6,
+ "seconds": 60,
+ "completed": 423,
+ "cancelled": 233,
+ "wrong": 0,
+ "errors": []
+ },
+ {
+ "slots": 16,
+ "seconds": 90,
+ "completed": 845,
+ "cancelled": 455,
+ "wrong": 0,
+ "errors": []
+ }
+ ],
+ "sampled": [
+ {
+ "streams": 1,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ]
+ ]
+ },
+ {
+ "streams": 4,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ]
+ ]
+ },
+ {
+ "streams": 8,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ],
+ [
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366"
+ ],
+ [
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff"
+ ],
+ [
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31"
+ ],
+ [
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf"
+ ]
+ ]
+ },
+ {
+ "streams": 16,
+ "hashes": [
+ [
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d",
+ "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d"
+ ],
+ [
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5",
+ "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5"
+ ],
+ [
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770",
+ "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770"
+ ],
+ [
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133",
+ "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133"
+ ],
+ [
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366",
+ "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366"
+ ],
+ [
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff",
+ "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff"
+ ],
+ [
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31",
+ "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31"
+ ],
+ [
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf",
+ "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf"
+ ],
+ [
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986",
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986",
+ "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986"
+ ],
+ [
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351",
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351",
+ "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351"
+ ],
+ [
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369",
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369",
+ "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369"
+ ],
+ [
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a",
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a",
+ "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a"
+ ],
+ [
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363",
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363",
+ "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363"
+ ],
+ [
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398",
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398",
+ "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398"
+ ],
+ [
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db",
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db",
+ "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db"
+ ],
+ [
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd",
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd",
+ "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd"
+ ]
+ ]
+ }
+ ],
+ "resume": {
+ "equal": true,
+ "cached_tokens": 3517,
+ "resumed_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91",
+ "fresh_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91"
+ },
+ "8bit": [
+ {
+ "streams": 1,
+ "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "concurrent_sha256": [
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58"
+ ]
+ },
+ {
+ "streams": 4,
+ "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "concurrent_sha256": [
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58"
+ ]
+ },
+ {
+ "streams": 8,
+ "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "concurrent_sha256": [
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58"
+ ]
+ },
+ {
+ "streams": 16,
+ "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "concurrent_sha256": [
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58",
+ "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58"
+ ]
+ },
+ {
+ "streams": 1,
+ "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "concurrent_sha256": [
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b"
+ ]
+ },
+ {
+ "streams": 4,
+ "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "concurrent_sha256": [
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b"
+ ]
+ },
+ {
+ "streams": 8,
+ "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "concurrent_sha256": [
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b"
+ ]
+ },
+ {
+ "streams": 16,
+ "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "concurrent_sha256": [
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b",
+ "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b"
+ ]
+ }
+ ],
+ "long_context": [
+ {
+ "length": 65536,
+ "plain_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "solo_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "concurrent_sha256": [
+ "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6",
+ "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6"
+ ]
+ },
+ {
+ "length": 130816,
+ "plain_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "solo_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "concurrent_sha256": [
+ "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3",
+ "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3"
+ ]
+ }
+ ],
+ "shutdowns": {
+ "tokens.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.5550237089999999,
+ "returncode": 0
+ },
+ "final-1-candidate-drafted.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.44267137500003173,
+ "returncode": 0
+ },
+ "final-0-published.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.3199985830000003,
+ "returncode": 0
+ },
+ "final-2-candidate-plain.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.3152450830000362,
+ "returncode": 0
+ },
+ "final-1-main.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.26646854199998415,
+ "returncode": 0
+ },
+ "final-2-candidate-drafted.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.7430550000000267,
+ "returncode": 0
+ },
+ "final-3-published.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.7333516659999759,
+ "returncode": 0
+ },
+ "final-3-candidate-plain.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.3819492080000373,
+ "returncode": 0
+ },
+ "final-2-main.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.4357252500000186,
+ "returncode": 0
+ },
+ "functional-fresh.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.551300375000011,
+ "returncode": 0
+ },
+ "final-0-candidate-plain.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.49287025000000995,
+ "returncode": 0
+ },
+ "functional-cache.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.5678107079999961,
+ "returncode": 0
+ },
+ "final-3-main.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.31679716700000426,
+ "returncode": 0
+ },
+ "final-0-main.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.3809736660000027,
+ "returncode": 0
+ },
+ "8bit-equality.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.48993237499996667,
+ "returncode": 0
+ },
+ "final-1-published.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.7285161659999915,
+ "returncode": 0
+ },
+ "long-serving.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.7801908750000166,
+ "returncode": 0
+ },
+ "final-3-candidate-drafted.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.5634543329999815,
+ "returncode": 0
+ },
+ "final-2-published.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.44117920899998353,
+ "returncode": 0
+ },
+ "final-1-candidate-plain.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.48965824999999086,
+ "returncode": 0
+ },
+ "final-0-candidate-drafted.shutdown": {
+ "forced_kill": false,
+ "seconds": 0.48779195900002037,
+ "returncode": 0
+ }
+ }
+ },
+ "host_checks": {
+ "focused": {
+ "steps_passed": 69,
+ "steps_total": 69,
+ "tests_passed": 117,
+ "tests_total": 117
+ },
+ "default_build": {
+ "steps_passed": 239,
+ "steps_total": 239
+ },
+ "stateful_proposer_regression": "passed",
+ "granted_depth_tests": 14,
+ "copy_tests": 34,
+ "lean": {
+ "main_findings": 425,
+ "candidate_findings": 425,
+ "new_findings": 0
+ }
+ },
+ "m4_qualification": {
+ "device": "Apple M4 Pro, 20 GPU cores, 48 GiB, macOS 26.5.1",
+ "operator_summary": "SUMMARY 11 kernels: 11 pass gates 1-4, 0 fail, 0 known findings; 17 controls: 17 red as planted, 0 missed; 0 limit problems; gates 66369 ms, limits 192 ms",
+ "drafter_reference": {
+ "picks_equal": 9,
+ "picks": 9,
+ "worst_confidence_difference": 0.00106,
+ "steady_rounds": 20,
+ "host_allocations": 0,
+ "resizes": 0
+ },
+ "serving_status": "stopped at the resumed-cache prerequisite: OS memory pressure explicitly disabled keeping prompt states; cached_tokens=0; stress/sample/8bit/memory phases did not run",
+ "pressure_warning": "one more stream only when none runs, no new kept prompt states",
+ "shutdown": {
+ "forced_kill": false,
+ "seconds": 0.3489787499420345,
+ "returncode": 0
+ },
+ "performance_status": "not qualified under the observed CPU/memory contention"
+ },
+ "load_memory": {
+ "device": "M5 Ultra",
+ "context": 4096,
+ "parallel": 1,
+ "before_generation": true,
+ "arms": {
+ "published": {
+ "context_window": 4096,
+ "context_fitted": false,
+ "process": {
+ "source": "darwin_rusage_info_v4",
+ "physical_footprint_bytes": 15696735176,
+ "lifetime_peak_physical_footprint_bytes": 15709645744
+ }
+ },
+ "candidate": {
+ "context_window": 4096,
+ "context_fitted": false,
+ "process": {
+ "source": "darwin_rusage_info_v4",
+ "physical_footprint_bytes": 15679974464,
+ "lifetime_peak_physical_footprint_bytes": 15733451768
+ }
+ }
+ },
+ "candidate_peak_delta_bytes": 23806024
+ },
+ "limitations": [
+ "A material four-stream code regression remains; the candidate is not a universal throughput win.",
+ "Candidate-plain four-stream prose also regresses versus main.",
+ "The additional shared calibration increases hot-model startup time by roughly one second in this run.",
+ "M4 served cache/concurrency qualification could not complete under system memory pressure; it is not recorded as passed.",
+ "M1/M2/M3/CUDA/HIP were not rerun; focused LFM2 operators are not the full cross-family kernel-contract gate."
+ ],
+ "load_readiness": [
+ {
+ "event": "ready",
+ "label": "final-0-main",
+ "seconds": 2.5573577919999906
+ },
+ {
+ "event": "ready",
+ "label": "final-0-published",
+ "seconds": 2.5535984580000104
+ },
+ {
+ "event": "ready",
+ "label": "final-0-candidate-plain",
+ "seconds": 3.573031125
+ },
+ {
+ "event": "ready",
+ "label": "final-0-candidate-drafted",
+ "seconds": 3.572747667000016
+ },
+ {
+ "event": "ready",
+ "label": "final-1-published",
+ "seconds": 2.5532073749999995
+ },
+ {
+ "event": "ready",
+ "label": "final-1-candidate-plain",
+ "seconds": 3.0623082079999904
+ },
+ {
+ "event": "ready",
+ "label": "final-1-candidate-drafted",
+ "seconds": 3.564061958999986
+ },
+ {
+ "event": "ready",
+ "label": "final-1-main",
+ "seconds": 1.5315142910000077
+ },
+ {
+ "event": "ready",
+ "label": "final-2-candidate-plain",
+ "seconds": 3.063439207999977
+ },
+ {
+ "event": "ready",
+ "label": "final-2-candidate-drafted",
+ "seconds": 3.5726659590000054
+ },
+ {
+ "event": "ready",
+ "label": "final-2-main",
+ "seconds": 2.0429697499999975
+ },
+ {
+ "event": "ready",
+ "label": "final-2-published",
+ "seconds": 2.552820082999972
+ },
+ {
+ "event": "ready",
+ "label": "final-3-candidate-drafted",
+ "seconds": 3.5727163330000167
+ },
+ {
+ "event": "ready",
+ "label": "final-3-main",
+ "seconds": 2.042876958000022
+ },
+ {
+ "event": "ready",
+ "label": "final-3-published",
+ "seconds": 2.5485449579999795
+ },
+ {
+ "event": "ready",
+ "label": "final-3-candidate-plain",
+ "seconds": 3.054198165999992
+ }
+ ],
+ "review_fixes": {
+ "rejected_guard_commits_excluded_from_ancestry": [
+ "5d9fcf68f87dd46ff653c17a2fd937fd09074ce1",
+ "1b57e583855b552aa003004e80db702ac1e6ae4e",
+ "4fa02928a7484ccf7bc1a529936618e9489eb24e",
+ "b66a7629e60d7283d5a837f7b3d2632f03000896",
+ "6539b72beacf5bf911d5a5141f50da88cfcab4c6"
+ ]
+ }
+}
diff --git a/docs/receipts/lfm2-dspark-review/wide_matrix.py b/docs/receipts/lfm2-dspark-review/wide_matrix.py
new file mode 100644
index 0000000000..7f672c2ea8
--- /dev/null
+++ b/docs/receipts/lfm2-dspark-review/wide_matrix.py
@@ -0,0 +1,30 @@
+# Four-arm Metal serving comparison; reserve the GPU exclusively before running.
+import importlib.util,os,fcntl,threading,concurrent.futures as cf,hashlib
+from pathlib import Path
+s=importlib.util.spec_from_file_location('q',Path(__file__).with_name('qualification.py'));q=importlib.util.module_from_spec(s);s.loader.exec_module(q)
+root=Path(os.environ.get('TF_QUAL_CANDIDATE_ROOT',str(q.ROOT)));q.OUT=Path(os.environ.get('TF_QUAL_OUT','/tmp/tf-lfm2-wide-matrix'));q.OUT.mkdir(exist_ok=True)
+with open('/tmp/tensorfold-model.lock','a') as lock:
+ fcntl.flock(lock,fcntl.LOCK_EX)
+ q.save('source-hashes',{p:hashlib.sha256((root/p).read_bytes()).hexdigest() for p in ['zig/src/core/lanes/depth.zig','zig/src/core/lanes/drafted_chances_test.zig','zig/src/families/lfm2_moe/backend.zig','zig/src/families/lfm2_moe/dspark.zig','zig/src/families/lfm2_moe/dspark_block.zig']})
+ arms={'main':(Path(os.environ['TF_QUAL_MAIN']),'plain'),'published':(Path(os.environ['TF_QUAL_PUBLISHED']),'bf16'),'candidate-plain':(q.CAND,'plain'),'candidate-drafted':(q.CAND,'bf16')}
+ q.save('binaries',{name:hashlib.sha256(binary.read_bytes()).hexdigest() for name,(binary,_) in arms.items()})
+ results=[];refs={}
+ prompts=['Write a short Python function that computes the Fibonacci sequence and explain it.','Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.']
+ for round_ in range(4):
+ order=list(arms);offset=round_%len(order);order=order[offset:]+order[:offset]
+ for arm in order:
+ binary,mode=arms[arm]
+ label=f'final-{round_}-{arm}'
+ with q.Server(label,binary,mode,4096,16):
+ if 'memory pressure:' in (q.OUT/(label+'.server.log')).read_text(errors='replace'):raise RuntimeError('memory pressure prevents a valid concurrency qualification')
+ for prompt in prompts:
+ q.post({'prompt':prompt,'max_tokens':32})
+ for width in [1,4,8,16]:
+ gate=threading.Barrier(width)
+ def run(i):gate.wait();return q.post({'prompt':prompt,'max_tokens':256},True)
+ with cf.ThreadPoolExecutor(width) as ex:rows=list(ex.map(run,range(width)))
+ ref=refs.setdefault(prompt,q.digest(rows[0]));assert all(q.digest(r)==ref for r in rows)
+ if 'memory pressure:' in (q.OUT/(label+'.server.log')).read_text(errors='replace'):raise RuntimeError('memory pressure changed stream admission during qualification')
+ rate=width*255/(max(r['last'] for r in rows)-min(r['first'] for r in rows))
+ item={'round':round_,'arm':arm,'prompt':'code' if prompt.startswith('Write') else 'prose','streams':width,'tps':rate,'requests':rows};results.append(item);q.save('comparison',results);q.emit('comparison',**{k:v for k,v in item.items() if k!='requests'})
+ q.emit('comparison-done',cells=len(results))
diff --git a/docs/recipes/README.md b/docs/recipes/README.md
index 8017124388..0cfdf94f1b 100644
--- a/docs/recipes/README.md
+++ b/docs/recipes/README.md
@@ -22,7 +22,7 @@ For agent-assisted contributions, read the [TensorFold contributor skill](SKILL.
| Qwen3.6-35B-A3B | [MLX with MTP drafts, one-GPU CUDA](qwen3.6-moe.md) |
| Step-5-Preview | [Native Metal on one 256 GB Mac: 2-bit experts, MTP drafts, images](step-5-preview.md) |
| Kolibri-1 | [Native CUDA, block FP8, research checkpoint](kolibri-1.md) |
-| LFM2.5-24B-A2B | [Native Metal, MLX 4-bit, shared rounds](lfm2.5-24b-a2b.md) |
+| LFM2.5-24B-A2B | [Native Metal, MLX 4-bit, shared rounds, DSpark drafts](lfm2.5-24b-a2b.md) |
| Spark-X2.6-4B | [Native CUDA](spark-x2.6-4b-cuda.md), [native Metal, the bf16 release, contexts to 1M](spark-x2.6-4b-metal.md) |
## Capability floor
diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md
index 7b8b2cdf4b..a3fce056e5 100644
--- a/docs/recipes/lfm2.5-24b-a2b.md
+++ b/docs/recipes/lfm2.5-24b-a2b.md
@@ -41,3 +41,84 @@ the shared-round kernels on it. The M4 Pro is not yet measured on a quiet machin
Every reply was token for token equal before and after, and equal to its solo run and to `"draft": false`.
Cold prefill at 2k, 8k and 32k tokens and peak memory did not change.
+
+## DSpark drafts
+
+LiquidAI's DSpark drafter for this model is
+[`LiquidAI/LFM2.5-24B-A2B-DSpark`](https://huggingface.co/LiquidAI/LFM2.5-24B-A2B-DSpark/tree/e052e5b1dd103c38cdd4f8ddc068d7dea4586f55)
+at revision `e052e5b1dd103c38cdd4f8ddc068d7dea4586f55` (327.7M bf16 parameters, 655 MB). Its weights are under the
+[LFM Open License v1.0](https://huggingface.co/LiquidAI/LFM2.5-24B-A2B-DSpark/blob/e052e5b1dd103c38cdd4f8ddc068d7dea4586f55/LICENSE);
+TensorFold does not ship them, so read the license before use.
+
+```bash
+hf download LiquidAI/LFM2.5-24B-A2B-DSpark --revision e052e5b1dd103c38cdd4f8ddc068d7dea4586f55 \
+ config.json model.safetensors LICENSE README.md --local-dir "$HOME/models/LFM2.5-24B-A2B-DSpark"
+zig-out/native/bin/tensorfold-native serve "$MODEL_DIR" --port 8080 --parallel 16 \
+ --drafter "$HOME/models/LFM2.5-24B-A2B-DSpark"
+```
+
+Without `--drafter` the engine serves as before. `"draft": false` on a request, or `--no-drafts`, turns the drafter
+off; the lane core verifies every draft, so a drafted reply equals its `"draft": false` reply token for token.
+
+The drafter has no embedding and no head of its own:
+
+- Context: the target's rows after layers 2, 10, 18, 26 and 34 (the backend's taps, `lfm2_tap`), through `fc` and
+ `hidden_norm`, give each of its five layers' context keys and values (k_norm, then RoPE at the row's position).
+- Block: the pending token and mask tokens run five Qwen3 layers with non-causal attention over the context and the
+ block, then the target's head. A block is as long as the deepest chain its pass holds, at most the nine rows the
+ drafter was trained on.
+- Markov head: row i's logits plus `W2 W1[previous token]` (rank 256); draft i is their argmax.
+
+The drafter runs the released bf16 matrices as they are (`--drafter-bits 0`, the default). `--drafter-bits 8` instead
+quantizes them at load to 8-bit MLX affine codes in groups of 64, about 5% faster on one stream and 361 MiB of weights
+against 625 MiB; the server logs which mode runs, and 4 bits is refused. Both run on the family's kernels
+(bf16 on `lfm2_bf16_rows`, `lfm2_bf16_mma` and `lfm2_markov_scan_bf16`, 8-bit on `core/tile_qmm`); the q and k rows are reordered so that the target's half-split RoPE kernel gives the
+checkpoint's interleaved RoPE. Its own kernels are `lfm2_tap`, `lfm2_attn_block` and `lfm2_attn_block_merge`
+(non-causal block attention, the target's attention kernels with a template flag), `lfm2_markov_scan` and
+`lfm2_markov_pick` (a Markov step) and `lfm2_draft_ids` (a hold's drafts into the next verify's rows on the GPU, so
+the host does not wait for a hold that fits one pass). Each kept prompt state keeps the drafter's context too, so a
+drafting stream resumes prompts. The lane core prices drafts across the admitted stream counts.
+
+`tf-kernel-contract --only lfm2` checks these kernels. `tf-lfm2-dspark MODEL DRAFTER TOKENS [ROWS]` runs the drafter
+against an fp32 host reference of the released checkpoint (the same taps and the target's embedding and head),
+prints each draft against the reference's pick with the margin between the reference's two best tokens, and times
+the drafter's passes; a near tie can pick the other token.
+
+### Verify schedule
+
+The drafter also carries a confidence head: a linear map of a draft's final block row and `W1[previous token]`
+(2,304 inputs) to one logit, through a sigmoid (`lfm2_dspark_confidence`). It does not change the drafts.
+The verify rows follow it: a chain's chance at draft j is the product of its first j confidences. Each round the
+lane core's window allocator takes the likeliest drafts over all streams while the expected tokens a ms rise, so
+each stream gets its own verify length, and the next hold drafts as deep as the stream's grant. A hold still
+running on the GPU is priced by its lane's last confidences, so the host does not wait for it. A checkpoint
+without the head verifies by the core's depth rule, from each stream's measured acceptance by depth.
+
+A stream with a copy ahead verifies the copy and asks no hold, and a drafting stream prices its copy
+windows by its own copies. The lane core also weighs the hold pass itself: the drafter times holds of 1 to 16
+blocks at every admissible depth at load and corrects those prices toward each settled hold's GPU time.
+Prices interpolate neighboring measured depths, preserving the cost changes at kernel transitions.
+
+`tf-lfm2-dspark` also prints the confidence
+head against its fp32 reference and times shared holds of 2 to 16 streams.
+
+The loader reads the checkpoint one matrix at a time and keeps the released bf16 matrices by default, or the
+8-bit codes with `--drafter-bits 8`; a stream's drafter lane is one
+Metal buffer (its layers' keys and values, then its staged rows), and its bytes join the streams the load limit
+admits. At load the drafter logs its weights, scratch and a stream's bytes. `tf-lfm2-dspark` also counts host
+allocations over twenty served drafter rounds, which must be none.
+
+A hold runs on one concurrent encoder with a barrier only where a step reads an earlier step's writes: q, k and v,
+and gate and up, share one sums pass, and the blocks of a shared hold attend side by side. An 8-bit projection takes
+the scalar twin up to 8 rows, the window kernel past them where the GPU has no tensor units, and the tensor units
+from 9 rows (the head from 5) where it has them. Held blocks use one bf16 16-row group, then the existing
+64x64 tiles past 16 rows, avoiding a second weight pass for shared holds of 17 to 32 rows. Context ingestion
+keeps the original row kernel through 32 rows, preserving the persistent context projections.
+When one lane's stage of 128 verified rows fills, its absorb pass takes
+every lane's staged rows, so the lanes' stages fill together.
+
+Shared verification past the 16-row qualified window uses measured increments
+from the shared forward path through 32 rows, anchored at the 16-row window
+price. This calibrates the batch cost without widening a stream's qualified
+window. The wide-batch receipt records the throughput gains, the accepted
+four-stream tradeoff, startup/footprint cost and hardware coverage limits.
diff --git a/zig/build/metal_targets.zig b/zig/build/metal_targets.zig
index 8a16d7432e..df5a809e3f 100644
--- a/zig/build/metal_targets.zig
+++ b/zig/build/metal_targets.zig
@@ -294,6 +294,7 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s
.{ .name = "tf-lfm2-forward", .path = "zig/tests/lfm2_forward.zig", .about = "LFM2-MoE native teacher-forced logits" },
.{ .name = "tf-lfm2-bench", .path = "zig/tests/lfm2_bench.zig", .about = "LFM2-MoE prompt-chunk and one-row decode timing" },
.{ .name = "tf-lfm2-exact", .path = "zig/tests/lfm2_exact.zig", .about = "LFM2-MoE prompt chunks, windows and committed-state exactness" },
+ .{ .name = "tf-lfm2-dspark", .path = "zig/tests/lfm2_dspark.zig", .about = "LFM2's DSpark drafter on the GPU against an fp32 host reference of the released drafter" },
.{ .name = "tf-lfm2-fp64", .path = "zig/tests/lfm2_fp64.zig", .about = "LFM2 projection classes and attention on the tensor units, the tiles and the twin, each against float64" },
.{ .name = "tensorfold", .path = "zig/src/main.zig", .about = "The native engine's command line" },
.{ .name = "tf-nemotron-fixtures", .path = "zig/tests/nemotron_fixtures.zig", .about = "Nemotron kernels against the Python engine's captured ops" },
@@ -462,6 +463,8 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s
b.step("test-nemotron-round-depth-host", "Priced head depth, nonlanding levels and fixed-depth controls without devices").dependOn(&round_depth.step);
const lfm2_tests = b.addRunArtifact(b.addTest(.{ .root_module = engine, .filters = &.{"lfm2_moe"} }));
b.step("test-lfm2", "The LFM2-MoE family's tests: config, draws, state and its shared rounds on this Mac's GPU").dependOn(&lfm2_tests.step);
+ const lfm2_dspark = b.addRunArtifact(b.addTest(.{ .root_module = engine, .filters = &.{"lfm2 dspark:"} }));
+ b.step("test-lfm2-dspark", "LFM2's DSpark drafter and its taps: config, quantizer and kernels on this Mac's GPU").dependOn(&lfm2_dspark.step);
test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = lanes })).step);
// the fake fabric maps its windows with mmap: the Macs' RDMA, not Windows
if (target.result.os.tag != .windows) test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = fabric })).step);
diff --git a/zig/kernels/metal/lfm2/lfm2.metal b/zig/kernels/metal/lfm2/lfm2.metal
index af6a2a7372..2da8578970 100644
--- a/zig/kernels/metal/lfm2/lfm2.metal
+++ b/zig/kernels/metal/lfm2/lfm2.metal
@@ -232,7 +232,7 @@ inline void lfm2_stage(const thread uint4* pre, threadgroup uint* ks, threadgrou
}
// Every row's attention partials (one kernel for prompt and decode): a threadgroup a (chunk, kv head, 8 * RB rows).
-template // a simdgroup's RB row blocks share each staged K and V fragment, each block's arithmetic its own
+template // RB row blocks share each staged K and V fragment; FULL: all see [0, len + 1)
inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* keys, const device bfloat* values,
lfm2_attn_args p, device float* pm, device float* pl, device float* po, uint rows, uint3 tg,
uint tid, uint sg, uint lane, threadgroup uint (*ks)[HD * (KT / 2)],
@@ -242,8 +242,8 @@ inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* ke
const uint kh = tg.y, r0 = tg.x * 8 * RB, head = kh * (QH / KVH) + sg;
const uint qid = lane / 4, fm = (qid & 4) + ((lane / 2) % 4), fn = (qid & 2) * 2 + (lane % 2) * 2;
uint count[RB]; // this lane's rows' visible keys
- for (int b = 0; b < RB; ++b) count[b] = p.len + min(r0 + 8 * b + fm, rows - 1) + 1;
- const uint top = p.len + min(r0 + 8 * RB - 1, rows - 1) + 1; // the tile's most
+ for (int b = 0; b < RB; ++b) count[b] = FULL ? p.len + 1 : p.len + min(r0 + 8 * b + fm, rows - 1) + 1;
+ const uint top = FULL ? p.len + 1 : p.len + min(r0 + 8 * RB - 1, rows - 1) + 1; // the tile's most
if (chunk * PER + sub0 * SUB >= top) return;
vec q[RB][8];
for (int b = 0; b < RB; ++b)
@@ -340,7 +340,7 @@ inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* ke
for (int d = 0; d < 8; ++d) *(device float2*)(po + at * HD + 8 * d + fn) = O[b][d];
}
}
-template
+template
[[kernel]] void lfm2_attn_mma(const device bfloat* queries [[buffer(0)]], const device bfloat* keys [[buffer(1)]],
const device bfloat* values [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]],
device float* pm [[buffer(4)]], device float* pl [[buffer(5)]], device float* po [[buffer(6)]],
@@ -349,15 +349,16 @@ template
uint lane [[thread_index_in_simdgroup]]) {
threadgroup uint ks[2][HD * (KT / 2)];
threadgroup bfloat vs[2][KB * KP];
- lfm2_attn_rows(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs);
+ lfm2_attn_rows(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs);
}
template [[host_name("lfm2_attn_mma")]] [[kernel]] decltype(lfm2_attn_mma<1>) lfm2_attn_mma<1>;
template [[host_name("lfm2_attn_mma_r2")]] [[kernel]] decltype(lfm2_attn_mma<2>) lfm2_attn_mma<2>;
// The chunks merged in chunk order for (head, row): Y [rows, QH*HD] bf16.
+template
inline void lfm2_merge_row(const device float* pm, const device float* pl, const device float* po, lfm2_attn_args p,
device bfloat* y, uint head, uint r, uint lane) {
- const uint count = p.len + r + 1;
+ const uint count = FULL ? p.len + 1 : p.len + r + 1;
const uint per = PER;
const uint chunks = (count + per - 1) / per;
const size_t at = (size_t(r) * QH + head) * (CHUNKS * p.subs);
@@ -391,12 +392,15 @@ inline void lfm2_merge_row(const device float* pm, const device float* pl, const
out[lane] = static_cast(o0 / l);
out[lane + 32] = static_cast(o1 / l);
}
+template
[[kernel]] void lfm2_attn_merge(const device float* pm [[buffer(0)]], const device float* pl [[buffer(1)]],
const device float* po [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]],
device bfloat* y [[buffer(4)]], uint2 tg [[threadgroup_position_in_grid]],
uint lane [[thread_index_in_simdgroup]]) {
- lfm2_merge_row(pm, pl, po, p, y, tg.x, tg.y, lane);
+ lfm2_merge_row(pm, pl, po, p, y, tg.x, tg.y, lane);
}
+template [[host_name("lfm2_attn_merge")]] [[kernel]] decltype(lfm2_attn_merge) lfm2_attn_merge;
+template [[host_name("lfm2_attn_block_merge")]] [[kernel]] decltype(lfm2_attn_merge) lfm2_attn_merge;
// Router logits [rows, E] in fp32: a simdgroup an expert, lane l over inputs 4l..4l+3 of each 128, then simd_sum.
[[kernel]] void lfm2_route_logits(const device bfloat* x [[buffer(0)]], const device float* router [[buffer(1)]],
@@ -568,3 +572,11 @@ constant constexpr uint SORT_CHUNKS = 16; // thread (e, c) counts expert e's pi
best[2 * row + 1] = bad;
}
}
+
+// A drafter's tap: H's rows [rows, D] copied into TAPS at column `at.y` of rows `at.x` values apart, 8 a thread.
+[[kernel]] void lfm2_tap(const device bfloat* h [[buffer(0)]], device bfloat* taps [[buffer(1)]], constant uint2& at [[buffer(2)]],
+ uint2 i [[thread_position_in_grid]]) {
+ if (i.x >= D / 8) return;
+ const device vec* from = (const device vec*)(h + size_t(i.y) * D);
+ ((device vec*)(taps + size_t(i.y) * at.x + at.y))[i.x] = from[i.x];
+}
diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal
new file mode 100644
index 0000000000..7440805007
--- /dev/null
+++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal
@@ -0,0 +1,258 @@
+// DSpark's Markov step, compiled after lfm2.metal's text (its includes and constants).
+
+// DSpark's Markov scan, W2 as 8-bit codes (BF16 false) or bf16 as released (true): a threadgroup's best of 512 tokens.
+struct lfm2_markov_args { uint stride, rank, vocab, prev_stride; };
+constant constexpr uint MARKOV_SPAN = 512;
+
+// Eight bf16 in a uint4 as floats (a bf16 is the high half of its float).
+inline void lfm2_unpack8(uint4 v, thread float* f) {
+ for (int j = 0; j < 4; ++j) {
+ f[2 * j] = as_type(v[j] << 16);
+ f[2 * j + 1] = as_type(v[j] & 0xffff0000u);
+ }
+}
+template
+inline void lfm2_markov_scan_t(const device bfloat* logits, const device bfloat* w1, const device uint8_t* w2,
+ const device uint* prev, constant lfm2_markov_args& p, device float* best,
+ device uint* at, const device bfloat* w2s, const device bfloat* w2b, uint2 tg,
+ uint2 grid, uint sg, uint lane, threadgroup float* e, threadgroup float* sv,
+ threadgroup uint* si) {
+ const uint s = tg.y, t = sg * 32 + lane, id = prev[size_t(s) * p.prev_stride];
+ for (uint i = t; i < p.rank; i += 256) e[i] = id < p.vocab ? float(w1[size_t(id) * p.rank + i]) : 0.0f;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const device bfloat* row = logits + size_t(s) * p.stride;
+ float mv = -INFINITY;
+ uint mi = 0xffffffffu;
+ bool nan = false;
+ const uint first = tg.x * MARKOV_SPAN, last = min(first + MARKOV_SPAN, p.vocab);
+ for (uint v = first + sg; v < last; v += 8) {
+ float d = 0.0f;
+ for (uint i = 8 * lane; i < p.rank; i += 256) {
+ if (BF16) {
+ float f[8];
+ lfm2_unpack8(*(const device uint4*)((const device bfloat*)w2 + size_t(v) * p.rank + i), f);
+ for (int j = 0; j < 8; ++j) d = fma(f[j], e[i + j], d);
+ continue;
+ }
+ const size_t g = (size_t(v) * p.rank + i) / 64;
+ const float sc = w2s[g], bi = w2b[g];
+ const uchar4 lo = *(const device uchar4*)(w2 + size_t(v) * p.rank + i);
+ const uchar4 hi = *(const device uchar4*)(w2 + size_t(v) * p.rank + i + 4);
+ for (int j = 0; j < 4; ++j) d = fma(fma(sc, float(lo[j]), bi), e[i + j], d);
+ for (int j = 0; j < 4; ++j) d = fma(fma(sc, float(hi[j]), bi), e[i + 4 + j], d);
+ }
+ d = simd_sum(d);
+ const float sc = float(static_cast(float(row[v]) + float(static_cast(d))));
+ nan = nan || isnan(sc);
+ if (sc > mv || (mi == 0xffffffffu && sc == mv)) {
+ mv = sc;
+ mi = v;
+ }
+ }
+ if (lane == 0) {
+ sv[sg] = nan ? NAN : mv;
+ si[sg] = mi;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (t != 0) return;
+ float bv = sv[0];
+ uint bi = si[0];
+ for (uint g = 1; g < 8; ++g) {
+ if (isnan(bv) || isnan(sv[g])) bv = NAN;
+ else if (sv[g] > bv || (sv[g] == bv && si[g] < bi)) {
+ bv = sv[g];
+ bi = si[g];
+ }
+ }
+ best[size_t(s) * grid.x + tg.x] = bv;
+ at[size_t(s) * grid.x + tg.x] = bi;
+}
+
+[[kernel]] void lfm2_markov_scan(const device bfloat* logits [[buffer(0)]], const device bfloat* w1 [[buffer(1)]],
+ const device uint8_t* w2 [[buffer(2)]], const device uint* prev [[buffer(3)]],
+ constant lfm2_markov_args& p [[buffer(4)]], device float* best [[buffer(5)]],
+ device uint* at [[buffer(6)]], const device bfloat* w2s [[buffer(7)]],
+ const device bfloat* w2b [[buffer(8)]], uint2 tg [[threadgroup_position_in_grid]],
+ uint2 grid [[threadgroups_per_grid]], uint sg [[simdgroup_index_in_threadgroup]],
+ uint lane [[thread_index_in_simdgroup]]) {
+ threadgroup float e[256];
+ threadgroup float sv[8];
+ threadgroup uint si[8];
+ lfm2_markov_scan_t(logits, w1, w2, prev, p, best, at, w2s, w2b, tg, grid, sg, lane, e, sv, si);
+}
+
+// The same scan over the released bf16 W2 [vocab, rank] (buffers 7 and 8 unused).
+[[kernel]] void lfm2_markov_scan_bf16(const device bfloat* logits [[buffer(0)]], const device bfloat* w1 [[buffer(1)]],
+ const device uint8_t* w2 [[buffer(2)]], const device uint* prev [[buffer(3)]],
+ constant lfm2_markov_args& p [[buffer(4)]], device float* best [[buffer(5)]],
+ device uint* at [[buffer(6)]], uint2 tg [[threadgroup_position_in_grid]],
+ uint2 grid [[threadgroups_per_grid]], uint sg [[simdgroup_index_in_threadgroup]],
+ uint lane [[thread_index_in_simdgroup]]) {
+ threadgroup float e[256];
+ threadgroup float sv[8];
+ threadgroup uint si[8];
+ lfm2_markov_scan_t(logits, w1, w2, prev, p, best, at, w1, w1, tg, grid, sg, lane, e, sv, si);
+}
+
+// DSpark's Markov step pick: the scan's best over a stream's threadgroups (ties to the lowest token), as a Pick.
+[[kernel]] void lfm2_markov_pick(const device float* best [[buffer(0)]], const device uint* at [[buffer(1)]],
+ constant uint& groups [[buffer(2)]], device uint* picks [[buffer(3)]],
+ uint s [[threadgroup_position_in_grid]], uint t [[thread_index_in_threadgroup]]) {
+ threadgroup float tv[256];
+ threadgroup uint ti[256];
+ threadgroup uint tn[256];
+ float bv = -INFINITY;
+ uint bi = 0xffffffffu, nan = 0;
+ for (uint g = t; g < groups; g += 256) {
+ const float v = best[size_t(s) * groups + g];
+ const uint i = at[size_t(s) * groups + g];
+ if (isnan(v)) nan = 1;
+ else if (v > bv || (v == bv && i < bi)) {
+ bv = v;
+ bi = i;
+ }
+ }
+ tv[t] = bv;
+ ti[t] = bi;
+ tn[t] = nan;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint w = 128; w > 0; w >>= 1) {
+ if (t < w) {
+ const float v = tv[t + w];
+ const uint i = ti[t + w];
+ if (v > tv[t] || (v == tv[t] && i < ti[t])) {
+ tv[t] = v;
+ ti[t] = i;
+ }
+ tn[t] |= tn[t + w];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (t == 0) {
+ picks[2 * s] = ti[0];
+ picks[2 * s + 1] = tn[0];
+ }
+}
+
+// Held drafts from picks[i * stride] into ids[i]; a nonfinite or past-vocabulary pick is token 0 and sets bad.
+[[kernel]] void lfm2_draft_ids(const device uint* picks [[buffer(0)]], constant uint4& p [[buffer(1)]],
+ device uint* ids [[buffer(2)]], device atomic_uint* bad [[buffer(3)]],
+ uint i [[thread_position_in_grid]]) {
+ if (i >= p.y) return;
+ const uint token = picks[size_t(i) * p.x];
+ const bool ok = token < p.z && picks[size_t(i) * p.x + 1] == 0u;
+ ids[i] = ok ? token : 0u;
+ if (!ok) atomic_store_explicit(bad, 1u, memory_order_relaxed);
+}
+
+// DSpark's confidence logit of draft i of stream s: proj . [x row s * n + i, W1[previous token]] in fp32, no bias.
+struct lfm2_confidence_args { uint n, pass, vocab, rank, hidden; };
+[[kernel]] void lfm2_dspark_confidence(const device bfloat* x [[buffer(0)]], const device bfloat* w1 [[buffer(1)]],
+ const device uint* ids [[buffer(2)]], const device uint* picks [[buffer(3)]],
+ const device bfloat* proj [[buffer(4)]],
+ constant lfm2_confidence_args& p [[buffer(5)]], device float* out [[buffer(6)]],
+ uint2 tg [[threadgroup_position_in_grid]], uint t [[thread_index_in_threadgroup]],
+ uint sg [[simdgroup_index_in_threadgroup]],
+ uint lane [[thread_index_in_simdgroup]]) {
+ threadgroup float part[8];
+ const uint i = tg.x, s = tg.y;
+ // the previous token: the block's pending id for the first draft, else the Markov step's pick before it
+ const uint id = i == 0 ? ids[size_t(s) * p.n] : picks[2 * (size_t(i - 1) * p.pass + s)];
+ const device bfloat* row = x + (size_t(s) * p.n + i) * p.hidden;
+ float d = 0.0f;
+ for (uint k = t; k < p.hidden; k += 256) d = fma(float(row[k]), float(proj[k]), d);
+ if (id < p.vocab)
+ for (uint k = t; k < p.rank; k += 256) d = fma(float(w1[size_t(id) * p.rank + k]), float(proj[p.hidden + k]), d);
+ d = simd_sum(d);
+ if (lane == 0) part[sg] = d;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (t != 0) return;
+ float sum = 0.0f;
+ for (uint g = 0; g < 8; ++g) sum += part[g];
+ out[size_t(i) * p.pass + s] = sum;
+}
+
+// DSpark's bf16 linears (the released matrices as stored): y [rows, n] = x [rows, k] W [n, k]^T, fp32 sums, bf16 out.
+struct lfm2_bf16_args { uint rows, n, k; };
+constant constexpr uint BF16_ROWS = 16;
+
+// Few rows: a simdgroup a column of W over 16 rows, so W is read once a 16 rows. k a multiple of 8.
+[[kernel]] void lfm2_bf16_rows(const device bfloat* x [[buffer(0)]], const device bfloat* w [[buffer(1)]],
+ constant lfm2_bf16_args& p [[buffer(2)]], device bfloat* y [[buffer(3)]],
+ uint2 tg [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]],
+ uint lane [[thread_index_in_simdgroup]]) {
+ const uint col = tg.x * 8 + sg, r0 = tg.y * BF16_ROWS;
+ if (col >= p.n) return;
+ const uint rows = min(BF16_ROWS, p.rows - r0);
+ float acc[BF16_ROWS];
+ for (uint r = 0; r < BF16_ROWS; ++r) acc[r] = 0.0f;
+ const device bfloat* wr = w + size_t(col) * p.k;
+ const device bfloat* xr = x + size_t(r0) * p.k;
+ for (uint i = 8 * lane; i < p.k; i += 256) {
+ float wv[8];
+ lfm2_unpack8(*(const device uint4*)(wr + i), wv);
+ for (uint r = 0; r < BF16_ROWS; ++r) {
+ if (r >= rows) break;
+ float xv[8];
+ lfm2_unpack8(*(const device uint4*)(xr + size_t(r) * p.k + i), xv);
+ for (int j = 0; j < 8; ++j) acc[r] = fma(wv[j], xv[j], acc[r]);
+ }
+ }
+ for (uint r = 0; r < BF16_ROWS; ++r) {
+ const float sum = simd_sum(acc[r]);
+ if (lane == 0 && r < rows) y[size_t(r0 + r) * p.n + col] = static_cast(sum);
+ }
+}
+
+// Many rows: 64x64 tiles of y, four simdgroups of 32x32, k in steps of 32. k a multiple of 32.
+[[kernel]] void lfm2_bf16_mma(const device bfloat* x [[buffer(0)]], const device bfloat* w [[buffer(1)]],
+ constant lfm2_bf16_args& p [[buffer(2)]], device bfloat* y [[buffer(3)]],
+ uint2 tg [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]],
+ uint t [[thread_index_in_threadgroup]]) {
+ threadgroup bfloat xs[64 * 32];
+ threadgroup bfloat ws[64 * 32];
+ threadgroup float out[4][32 * 32];
+ const uint c0 = tg.x * 64, r0 = tg.y * 64;
+ simdgroup_matrix acc[4][4];
+ for (uint i = 0; i < 4; ++i)
+ for (uint j = 0; j < 4; ++j) acc[i][j] = simdgroup_matrix(0.0f);
+ const uint sr = (sg / 2) * 32, sc = (sg % 2) * 32;
+ for (uint k0 = 0; k0 < p.k; k0 += 32) {
+ for (uint i = t; i < 64 * 4; i += 128) { // 64 rows of 32 bf16 as 4 chunks of 8, 16 bytes a load
+ const uint r = i / 4, q = (i % 4) * 8;
+ const uint4 xv = r0 + r < p.rows ? *(const device uint4*)(x + size_t(r0 + r) * p.k + k0 + q) : uint4(0);
+ const uint4 wv = c0 + r < p.n ? *(const device uint4*)(w + size_t(c0 + r) * p.k + k0 + q) : uint4(0);
+ *(threadgroup uint4*)(xs + r * 32 + q) = xv;
+ *(threadgroup uint4*)(ws + r * 32 + q) = wv;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint kk = 0; kk < 32; kk += 8) {
+ simdgroup_matrix a[4], b[4];
+ for (uint i = 0; i < 4; ++i) simdgroup_load(a[i], xs + (sr + 8 * i) * 32 + kk, 32);
+ for (uint j = 0; j < 4; ++j) simdgroup_load(b[j], ws + (sc + 8 * j) * 32 + kk, 32, ulong2(0, 0), true);
+ for (uint i = 0; i < 4; ++i)
+ for (uint j = 0; j < 4; ++j) simdgroup_multiply_accumulate(acc[i][j], a[i], b[j], acc[i][j]);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ for (uint i = 0; i < 4; ++i)
+ for (uint j = 0; j < 4; ++j) simdgroup_store(acc[i][j], out[sg] + 8 * i * 32 + 8 * j, 32);
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint i = t % 32; i < 32 * 32; i += 32) {
+ const uint r = i / 32, c = i % 32;
+ const uint row = r0 + sr + r, col = c0 + sc + c;
+ if (row < p.rows && col < p.n) y[size_t(row) * p.n + col] = static_cast(out[sg][i]);
+ }
+}
+
+// A drafter block's attention: lfm2_attn_mma where every row sees all keys, its 128 threads reserved for M1 and M2.
+[[max_total_threads_per_threadgroup(128)]] [[kernel]] void lfm2_attn_block(
+ const device bfloat* queries [[buffer(0)]], const device bfloat* keys [[buffer(1)]],
+ const device bfloat* values [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]], device float* pm [[buffer(4)]],
+ device float* pl [[buffer(5)]], device float* po [[buffer(6)]], constant uint& rows [[buffer(7)]],
+ uint3 tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ uint sg [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]]) {
+ threadgroup uint ks[2][HD * (KT / 2)];
+ threadgroup bfloat vs[2][KB * KP];
+ lfm2_attn_rows<1, true>(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs);
+}
diff --git a/zig/kernels/metal/sources.zig b/zig/kernels/metal/sources.zig
index 2506fb4905..9f4604f678 100644
--- a/zig/kernels/metal/sources.zig
+++ b/zig/kernels/metal/sources.zig
@@ -19,7 +19,7 @@ pub const gemma4_prompt_attention = @embedFile("gemma4/prompt_attention.metal");
pub const gemma4_vision = @embedFile("prefill/gemm_nax.metal") ++ @embedFile("gemma4/vision.metal");
/// LFM2-MoE: embedding, norms, the short conv, attention, the route and the experts' glue.
-pub const lfm2 = @embedFile("lfm2/lfm2.metal") ++ @embedFile("lfm2/lfm2_segments.metal");
+pub const lfm2 = @embedFile("lfm2/lfm2.metal") ++ @embedFile("lfm2/lfm2_dspark.metal") ++ @embedFile("lfm2/lfm2_segments.metal");
/// LFM2's prompt attention on the tensor units (nax.h inlined at load, after lfm2.metal's text).
pub const lfm2_attn_nax = @embedFile("lfm2/lfm2_attn_nax.metal");
diff --git a/zig/src/core/contract/lfm2_records.zig b/zig/src/core/contract/lfm2_records.zig
index 355559b1ff..2e7587fa3d 100644
--- a/zig/src/core/contract/lfm2_records.zig
+++ b/zig/src/core/contract/lfm2_records.zig
@@ -1,12 +1,23 @@
//! LFM2-MoE's shared-round and greedy-pick launches (lfm2.metal, lfm2_segments.metal), as the encoders launch them.
const std = @import("std");
const M = @import("limits.zig").MetalLaunch;
+// The drafter kernels carry M1 and M2 limits too, recorded from the offline compile; no run covers those chips.
pub const records = [_]M{
.{ .name = "metal lfm2 argmax lfm2_argmax", .threads = 1024, .tg_memory = 384, .m1 = 1024, .m2 = 1024 },
.{ .name = "metal lfm2 stream conv lfm2_conv_segs", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 },
.{ .name = "metal lfm2 stream attention qk lfm2_qk_segs", .threads = 32, .tg_memory = 0, .m1 = 832, .m2 = 832 },
.{ .name = "metal lfm2 stream attention lfm2_attn_segs", .threads = 128, .tg_memory = 18688, .reserved = 128 },
.{ .name = "metal lfm2 stream attention merge lfm2_attn_merge_segs", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 },
+ .{ .name = "metal lfm2 tap lfm2_tap", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 },
+ .{ .name = "metal lfm2 markov scan lfm2_markov_scan", .threads = 256, .tg_memory = 1088, .m1 = 1024, .m2 = 1024 },
+ .{ .name = "metal lfm2 markov scan bf16 lfm2_markov_scan_bf16", .threads = 256, .tg_memory = 1088, .m1 = 1024, .m2 = 1024 },
+ .{ .name = "metal lfm2 bf16 rows lfm2_bf16_rows", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 },
+ .{ .name = "metal lfm2 bf16 tile lfm2_bf16_mma", .threads = 128, .tg_memory = 24576, .m1 = 832, .m2 = 832 },
+ .{ .name = "metal lfm2 markov pick lfm2_markov_pick", .threads = 256, .tg_memory = 3072, .m1 = 1024, .m2 = 1024 },
+ .{ .name = "metal lfm2 draft ids lfm2_draft_ids", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 },
+ .{ .name = "metal lfm2 block attention lfm2_attn_block", .threads = 128, .tg_memory = 18688, .reserved = 128 },
+ .{ .name = "metal lfm2 block attention merge lfm2_attn_block_merge", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 },
+ .{ .name = "metal lfm2 dspark confidence lfm2_dspark_confidence", .threads = 256, .tg_memory = 32, .m1 = 1024, .m2 = 1024 },
};
test "the LFM2 launches fit every Apple family" {
diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig
index 0f0905212f..38b1d9be0a 100644
--- a/zig/src/core/engine_api.zig
+++ b/zig/src/core/engine_api.zig
@@ -303,10 +303,12 @@ pub const Open = struct {
/// --parallel named a number: an engine whose memory fits fewer streams refuses instead of serving fewer.
lanes_fixed: bool = false,
drafts: bool = true,
- /// --drafter: a draft model's directory (Qwen3.8-27B's DFlash2; null: none), and --drafter-bits: 4, or 0 for bf16.
+ /// --drafter: a draft model's directory (null: none); --drafter-bits: 4, 0 for bf16, or 8 (LFM2 only).
drafter: ?[]const u8 = null,
drafter_name: ?[]const u8 = null, // the supplied --drafter name, before resolving its directory
drafter_bits: u8 = 4,
+ /// Whether --drafter-bits was given: LFM2's drafter runs bf16 as released unless it is asked for 8-bit codes.
+ drafter_bits_given: bool = false,
/// --record: each stream's committed rows and tapped features written under this folder, for drafter training.
record: ?[]const u8 = null,
speed_up: ?[]const u8 = null,
diff --git a/zig/src/core/lanes/allocate.zig b/zig/src/core/lanes/allocate.zig
index c3c592af60..d15b489c55 100644
--- a/zig/src/core/lanes/allocate.zig
+++ b/zig/src/core/lanes/allocate.zig
@@ -81,12 +81,17 @@ fn rate(costs: *const Table, timed: bool, total: u64, expected: f64, overhead_ms
/// A chain's nodes' chances from per-depth acceptance (the j-th lands if every earlier one did). Caller frees.
pub fn chainProbabilities(gpa: Allocator, rates: []const f64, count: usize) ![]f64 {
const out = try gpa.alloc(f64, count);
+ chainProbabilitiesInto(rates, out);
+ return out;
+}
+
+/// Fill a caller-owned chain buffer without allocating a second copy.
+pub fn chainProbabilitiesInto(rates: []const f64, out: []f64) void {
var reach: f64 = 1.0;
for (out, 0..) |*o, j| {
reach *= if (j < rates.len) rates[j] else rates[rates.len - 1];
o.* = reach;
}
- return out;
}
/// Costs for 1..rows: timed widths as measured, interpolated between them, the widest's ms a row past it.
diff --git a/zig/src/core/lanes/config.zig b/zig/src/core/lanes/config.zig
index a6721fcf57..deb76d4be2 100644
--- a/zig/src/core/lanes/config.zig
+++ b/zig/src/core/lanes/config.zig
@@ -36,7 +36,11 @@ pub const Model = struct {
max_streams: u32 = 0, // a shared forward's streams at most (0: 32)
shared_costs: []const Cost = &.{}, // shared forwards' ms by total rows (timed at load)
draft_probabilities: bool = false, // the head gives each drafted node's chance of landing
+ granted_depth: bool = false, // with draft_probabilities: every stream drafts its last grant plus a slack
+ draft_slack: u32 = 0, // drafts past a stream's last grant, short or not (0: the Constants' own)
draft_streams: bool = false, // the head drafts every stream of a shared round in one batch
+ copy_first: bool = false, // a copy ahead takes the round: the head holds no drafts for it (a drafter's own pass)
+ hold_costs: []const @import("hold_price.zig").HoldCost = &.{}, // a drafter's own hold pass, timed at load
head_trees: bool = false, // the head drafts trees of lanes (DraftRequest.lanes) and holds them
join_tail: bool = false, // a prompt in pieces leaves its last token for the next shared round's window (Backend.join)
piece_runs: bool = false, // a round's pieces go to the backend in one call (Backend.pieces: one forward over them)
@@ -66,6 +70,7 @@ pub const Config = struct {
batch_rows: u32,
batch_streams: u32,
node_probabilities: bool,
+ granted_depth: bool = false, // Model.granted_depth
fill_lanes: u32 = 0, // rows a window may grow to with suffix-match branches (0: off; lanes/fill.zig)
fill_match: u32 = 8, // context tokens a suffix-match branch must match (shorter ones are coincidental phrases)
fill_floor: f64 = 0.35, // the share of grafted rows a stream must keep to go on reading its chain back every round
@@ -149,6 +154,7 @@ pub const Config = struct {
.batch_rows = batch_rows,
.batch_streams = if (streams) (if (m.max_streams != 0) m.max_streams else 32) else 1,
.node_probabilities = m.draft_probabilities or m.draft_confidence != null,
+ .granted_depth = m.granted_depth and m.draft_probabilities,
.draft_streams = m.draft_streams,
.head_trees = m.head_trees,
.join_tail = m.join_tail,
@@ -160,7 +166,14 @@ pub const Config = struct {
.shared_costs = try alloc.extendCosts(gpa, timed.items, batch_rows),
.lane_costs = try alloc.extendCosts(gpa, m.lane_costs, 64),
.mtp_step_ms = m.mtp_step_ms,
- .k = .{ .depth_rate = if (m.depth_rate > 0) m.depth_rate else (Constants{}).depth_rate },
+ .k = .{
+ .depth_rate = if (m.depth_rate > 0) m.depth_rate else (Constants{}).depth_rate,
+ .draft_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).draft_slack,
+ .grant_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).grant_slack,
+ .short_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).short_slack,
+ .copy_first = m.copy_first and !m.context_copies,
+ .hold_costs = m.hold_costs,
+ },
.draft_step_ms = m.draft_step_ms,
};
}
diff --git a/zig/src/core/lanes/depth.zig b/zig/src/core/lanes/depth.zig
index 8595a67aab..efb8dfa018 100644
--- a/zig/src/core/lanes/depth.zig
+++ b/zig/src/core/lanes/depth.zig
@@ -12,8 +12,15 @@ pub const Constants = struct {
plain_margin: f64 = 1.05, // a drafted depth must beat plain rounds' tokens a ms by this much
plain_wait_most: u64 = 128, // plain rounds between probes at most
draft_slack: i64 = 2, // drafts a stream offers past what its last shared round granted it
+ grant_slack: i64 = 1, // granted depths: drafts past a grant that verified every draft the stream held
+ short_slack: i64 = 0, // granted depths: drafts past a grant that left some of them unverified
+ copy_first: bool = false, // a copy ahead takes the next round whole: the head drafts none for it (else one)
+ hold_costs: []const HoldCost = &.{}, // a drafter's own hold pass by blocks and depth: granted holds must pay for it
};
+pub const HoldCost = @import("hold_price.zig").HoldCost;
+const holdMs = @import("hold_price.zig").holdMs;
+
/// A stream's acceptance by depth and probe counters (Python `_depth_state[stream]`).
pub const State = struct {
p: []f64,
@@ -35,6 +42,8 @@ pub const Who = struct {
finished: bool,
forced: bool,
granted: ?u32 = null,
+ grant_short: bool = false, // the last grant left some held drafts unverified (the width did not bind)
+ gain: ?f64 = null, // the drafts its last grant expected to keep (null: none priced)
stream: ?*anyopaque = null,
sampled: bool = false, // keyed draws: host work a row that a greedy round does not have
};
@@ -59,6 +68,7 @@ pub const Rule = struct {
draft_step_ms: f64 = 0, // a shared round's draft past a stream's first (budgets price it)
plain_guard: bool,
node_probabilities: bool,
+ granted_depth: bool = false, // node_probabilities: a stream drafts its last grant plus draft_slack, lone ones too
confidence_floor: ?f64 = null,
batch_rows: u32,
batched_heads: bool = false, // one batch drafts every stream of a shared round (its steps: the deepest stream's)
@@ -68,7 +78,10 @@ pub const Rule = struct {
shared_rounds: u64 = 0,
greedy_overhead: std.ArrayList(Overhead) = .empty, // overhead from rounds without a sampled stream
+ hold_order: []usize = &.{}, // reused hold ordering, at most batch_rows entries
+
pub fn deinit(r: *Rule) void {
+ r.gpa.free(r.hold_order);
r.round_ms.deinit(r.gpa);
r.overhead.deinit(r.gpa);
r.greedy_ms.deinit(r.gpa);
@@ -192,17 +205,105 @@ pub const Rule = struct {
return true;
}
- /// Python `_head_depth`: the next round's head drafts for a stream (0 when forced, 1 with a copy ahead).
+ /// Drafts past a grant: none when it stopped short of the held drafts (the width did not bind), else grant_slack.
+ fn slack(r: *const Rule, who: Who) i64 {
+ return if (who.grant_short) r.k.short_slack else r.k.grant_slack;
+ }
+
+ /// The last grant plus its slack (the deepest chain before any); after a grant of none, a probe of draft_slack.
+ fn grantDepth(r: *Rule, who: Who) !i64 {
+ const most = @min(@as(i64, r.most_drafts), @max(1, who.draft_room - 1));
+ const granted = who.granted orelse return most;
+ _ = try r.rates(who);
+ const st = &who.state.*.?;
+ if (granted == 0) return if (!r.plain_guard or r.probePlain(st)) @min(most, r.k.draft_slack) else 0;
+ st.plain = 0;
+ st.wait = r.k.depth_probe_every;
+ return @min(most, @as(i64, granted) + r.slack(who));
+ }
+
+ /// Python `_head_depth`: the next round's head drafts for a stream (0 when forced, 1 or none with a copy ahead).
pub fn headDepth(r: *Rule, who: *const Who, budget: ?i64, probe: Probe) !i64 {
- const d: i64 = if (who.finished) 0 else if (budget) |b| b else if (r.node_probabilities)
+ const d: i64 = if (who.finished) 0 else if (budget) |b| b else if (r.granted_depth)
+ try r.grantDepth(who.*)
+ else if (r.node_probabilities)
@min(@as(i64, r.most_drafts), @max(1, who.draft_room - 1))
else
try r.depth(who.*);
if (who.forced) return 0;
- if (d != 0 and probe.copy(probe.ptr, who)) return 1;
+ if (d != 0 and probe.copy(probe.ptr, who)) return @intFromBool(!r.k.copy_first);
return d;
}
+ /// A shared hold pads every block to its deepest: depths cut so the drafting streams' blocks fit `rows` (one pass).
+ fn capPass(out: []i64, rows: u32) void {
+ var n: i64 = 0;
+ for (out) |d| n += @intFromBool(d > 0);
+ if (n == 0) return;
+ const most = @max(1, @divFloor(@as(i64, rows), n));
+ for (out) |*d| d.* = @min(d.*, most);
+ }
+
+ /// Granted holds kept, most expected drafts a row first, while they raise the next round's expected tokens a ms.
+ fn payHolds(r: *Rule, whos: []const Who, sampled: bool, out: []i64) !void {
+ if (whos.len > r.batch_rows) return error.BadHoldStreamCount;
+ if (r.hold_order.len == 0) r.hold_order = try r.gpa.alloc(usize, r.batch_rows);
+ const order = r.hold_order[0..whos.len];
+ var m: usize = 0;
+ var probes: usize = 0; // a probe after a grant of none keeps its hold: it measures the stream again
+ var deepest: i64 = 0;
+ for (whos, out, 0..) |w, d, i| {
+ if (d <= 0) continue;
+ if (w.granted == @as(?u32, 0)) {
+ probes += 1;
+ deepest = @max(deepest, d);
+ } else {
+ order[m] = i;
+ m += 1;
+ }
+ }
+ if (m == 0) return;
+ const Ctx = struct {
+ r: *Rule,
+ whos: []const Who,
+ fn less(c: @This(), x: usize, y: usize) bool {
+ const gx = c.r.gainOf(c.whos[x]) catch 0;
+ const gy = c.r.gainOf(c.whos[y]) catch 0;
+ return gx * rowsOf(c.whos[y]) > gy * rowsOf(c.whos[x]) or (gx * rowsOf(c.whos[y]) == gy * rowsOf(c.whos[x]) and x < y);
+ }
+ };
+ std.mem.sort(usize, order[0..m], Ctx{ .r = r, .whos = whos }, Ctx.less);
+ const fixed = r.fixedMs(whos.len, sampled);
+ var rows: f64 = @floatFromInt(whos.len);
+ var tokens = rows;
+ const plain = r.shared_costs.get(@intCast(whos.len)) orelse return;
+ const probe_ms = if (probes > 0) holdMs(r.k.hold_costs, probes, deepest) else 0;
+ var best = tokens / (plain + fixed + probe_ms);
+ var keep: usize = 0;
+ for (order[0..m], 1..) |i, k| {
+ rows += rowsOf(whos[i]);
+ tokens += try r.gainOf(whos[i]);
+ deepest = @max(deepest, out[i]);
+ const verify = r.shared_costs.get(@intFromFloat(@ceil(rows))) orelse break;
+ const now = tokens / (verify + fixed + holdMs(r.k.hold_costs, k + probes, deepest));
+ if (now > best) {
+ best = now;
+ keep = k;
+ }
+ }
+ for (order[keep..m]) |i| out[i] = 0;
+ }
+
+ /// The drafts a granted stream's next round is expected to keep: its last grant's, or its first draft's rate.
+ fn gainOf(r: *Rule, w: Who) !f64 {
+ if (w.gain) |g| if (w.granted != null) return g;
+ return (try r.rates(w))[0];
+ }
+
+ fn rowsOf(w: Who) f64 {
+ return @floatFromInt(@max(1, w.granted orelse 1));
+ }
+
/// Head drafts for the streams of a shared round, sized by allocation over their chain chances.
pub fn budgets(r: *Rule, whos: []Who, probe: Probe, sampled: bool, out: []i64) !void {
if (r.confidence_floor != null) {
@@ -213,8 +314,10 @@ pub const Rule = struct {
for (whos, out) |*w, *o| {
const inner = try r.headDepth(w, null, probe);
const granted: i64 = if (w.granted) |g| g else r.most_drafts;
- o.* = try r.headDepth(w, @min(inner, granted + r.k.draft_slack), probe);
+ o.* = if (r.granted_depth) inner else try r.headDepth(w, @min(inner, granted + r.k.draft_slack), probe);
}
+ if (r.granted_depth) capPass(out, r.batch_rows);
+ if (r.granted_depth and r.k.hold_costs.len > 0) try r.payHolds(whos, sampled, out);
return;
}
const most = try r.gpa.alloc(i64, whos.len);
@@ -381,3 +484,88 @@ test "a shared round's drafts come out the same whichever depths its overhead wa
}
try std.testing.expectEqualSlices(i64, &got[0], &got[1]);
}
+
+test "granted depths in a shared round fit one hold pass: every block is as deep as the deepest" {
+ const empty: Table = .{};
+ var rule: Rule = .{ .gpa = std.testing.allocator, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = false, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 };
+ defer rule.deinit();
+ var states: [8]?State = @splat(null);
+ defer for (&states) |*st| if (st.*) |*x| x.deinit(std.testing.allocator);
+ var whos: [8]Who = undefined;
+ for (&whos, &states, 0..) |*w, *st, i| w.* = .{ .state = st, .draft_room = 100, .finished = false, .forced = false, .granted = @intCast(i) };
+ const No = struct {
+ fn copy(_: *anyopaque, _: *const Who) bool {
+ return false;
+ }
+ };
+ var unused: u8 = 0;
+ var out: [8]i64 = undefined;
+ try rule.budgets(&whos, .{ .ptr = &unused, .copy = No.copy }, false, &out);
+ // a grant of none probes two; grants 1..7 that took every held draft, plus one; cut to 32 rows over eight blocks
+ try std.testing.expectEqualSlices(i64, &.{ 2, 2, 3, 4, 4, 4, 4, 4 }, &out);
+ try rule.budgets(whos[0..2], .{ .ptr = &unused, .copy = No.copy }, false, out[0..2]);
+ try std.testing.expectEqualSlices(i64, &.{ 2, 2 }, out[0..2]); // two blocks fit whole
+ for (whos[0..2]) |*w| w.grant_short = true;
+ try rule.budgets(whos[0..2], .{ .ptr = &unused, .copy = No.copy }, false, out[0..2]);
+ try std.testing.expectEqualSlices(i64, &.{ 2, 1 }, out[0..2]); // a grant short of the held drafts: no slack
+}
+
+test "a copy ahead: one head draft beside it, or none when the head's drafts cost a pass of their own" {
+ const empty: Table = .{};
+ const Yes = struct {
+ fn copy(_: *anyopaque, _: *const Who) bool {
+ return true;
+ }
+ };
+ var unused: u8 = 0;
+ const probe: Probe = .{ .ptr = &unused, .copy = Yes.copy };
+ var st: ?State = null;
+ defer if (st) |*x| x.deinit(std.testing.allocator);
+ const who: Who = .{ .state = &st, .draft_room = 100, .finished = false, .forced = false, .granted = 3 };
+ for ([_]bool{ false, true }) |first| {
+ var rule: Rule = .{ .gpa = std.testing.allocator, .k = .{ .copy_first = first }, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = false, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 };
+ defer rule.deinit();
+ try std.testing.expectEqual(@as(i64, @intFromBool(!first)), try rule.headDepth(&who, null, probe));
+ var out: [1]i64 = undefined;
+ var whos = [_]Who{who};
+ try rule.budgets(&whos, probe, false, &out);
+ try std.testing.expectEqual(@as(i64, @intFromBool(!first)), out[0]);
+ }
+}
+
+test "a granted hold stays only while its stream's expected drafts pay for the verify rows and the hold pass" {
+ const gpa = std.testing.allocator;
+ var shared: Table = .{};
+ defer shared.deinit(gpa);
+ for (1..33) |w| try shared.put(gpa, @intCast(w), 10.0 + @as(f64, @floatFromInt(w)));
+ const empty: Table = .{};
+ const cheap = [_]HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1.0 }, .{ .streams = 8, .depth = 1, .ms = 4.5 } };
+ var states: [4]?State = @splat(null);
+ defer for (&states) |*st| if (st.*) |*x| x.deinit(gpa);
+ const granted = [_]?u32{ 2, 2, 0, 1 };
+ const gains = [_]?f64{ 1.8, 0.2, null, 0.9 };
+ var whos: [4]Who = undefined;
+ for (&whos, &states, granted, gains) |*w, *st, g, x| w.* = .{ .state = st, .draft_room = 100, .finished = false, .forced = false, .granted = g, .gain = x };
+ var rule: Rule = .{ .gpa = gpa, .k = .{ .hold_costs = &cheap }, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &shared, .mtp_step_ms = 0.5, .plain_guard = true, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 };
+ defer rule.deinit();
+ // 8 ms overhead: plain 4 / 22; stream 0 then 3 raise the rate (5.8 / 25, 6.7 / 26.5), stream 1 would lower it
+ var out = [_]i64{ 4, 4, 0, 3 };
+ try rule.payHolds(&whos, false, &out);
+ try std.testing.expectEqualSlices(i64, &.{ 4, 0, 0, 3 }, &out);
+ var failing = std.testing.FailingAllocator.init(gpa, .{ .fail_index = 0 });
+ rule.gpa = failing.allocator();
+ out = .{ 4, 4, 0, 3 };
+ try rule.payHolds(&whos, false, &out);
+ try std.testing.expect(!failing.has_induced_failure);
+ try std.testing.expectEqualSlices(i64, &.{ 4, 0, 0, 3 }, &out);
+ rule.gpa = gpa;
+ const dear = [_]HoldCost{.{ .streams = 1, .depth = 1, .ms = 20.0 }};
+ rule.k.hold_costs = &dear;
+ out = .{ 4, 4, 0, 3 };
+ try rule.payHolds(&whos, false, &out);
+ try std.testing.expectEqualSlices(i64, &.{ 0, 0, 0, 0 }, &out); // no gain pays a 20 ms pass
+ whos[2].granted = 0;
+ out = .{ 4, 4, 2, 3 };
+ try rule.payHolds(&whos, false, &out);
+ try std.testing.expectEqualSlices(i64, &.{ 0, 0, 2, 0 }, &out); // a probe after a grant of none keeps its hold
+}
diff --git a/zig/src/core/lanes/drafted.zig b/zig/src/core/lanes/drafted.zig
index b7b9ca5bc6..6c9f7edc4b 100644
--- a/zig/src/core/lanes/drafted.zig
+++ b/zig/src/core/lanes/drafted.zig
@@ -105,6 +105,15 @@ pub const Drafted = struct {
m.draft_streams = d.batched;
m.draft_probabilities = false;
m.head_trees = d.trees;
+ m.copy_first = d.copy_first;
+ m.hold_costs = d.hold_costs;
+ if (d.chances and x.drafter.vtable.chances != null) {
+ // a chain's drafts priced by the drafter's own chances; the next hold as deep as the grant and its slack
+ m.draft_probabilities = d.confidence_floor == null;
+ m.draft_confidence = d.confidence_floor;
+ m.granted_depth = d.confidence_floor == null;
+ m.draft_slack = d.slack;
+ }
if (d.tree_nodes > 0) {
// the drafter grows its own trees, best first, and prices their nodes
m.drafts = @min(d.tree_nodes, if (target.exact_width > 1) target.exact_width - 1 else 0);
diff --git a/zig/src/core/lanes/drafted_chances_test.zig b/zig/src/core/lanes/drafted_chances_test.zig
new file mode 100644
index 0000000000..233ec38fe5
--- /dev/null
+++ b/zig/src/core/lanes/drafted_chances_test.zig
@@ -0,0 +1,287 @@
+//! A drafter that prices its chains (Facts.chances): verify rows go by its chances and replies stay one-token decode.
+const std = @import("std");
+const Config = @import("config.zig").Config;
+const Model = @import("config.zig").Model;
+const Cost = @import("config.zig").Cost;
+const Engine = @import("engine.zig").Engine;
+const be = @import("backend.zig");
+const sm = @import("stream.zig");
+const fake = @import("fake.zig");
+const dr = @import("drafter.zig");
+const depth = @import("depth.zig");
+const Table = @import("table.zig").Table;
+const Drafted = @import("drafted.zig").Drafted;
+const dt = @import("drafted_test.zig");
+
+const gpa = std.testing.allocator;
+const most_streams = 8;
+
+/// drafted_test's drafter with a chance for each held draft: `sure` where its guess is right, `unsure` where wrong.
+const Priced = struct {
+ head: dt.FakeDrafter,
+ sure: f64 = 0.7,
+ unsure: f64 = 0.7,
+ vtable: dr.Drafter.VTable = undefined,
+ inner_hold: *const fn (*anyopaque, []const dr.Hold) anyerror!void = undefined,
+ at: std.AutoHashMapUnmanaged(*sm.Stream, u64) = .empty, // each stream's last hold's position
+ asked: u32 = 0,
+ held: u64 = 0, // drafts its holds asked for
+
+ fn of(ptr: *anyopaque) *Priced {
+ return @fieldParentPtr("head", @as(*dt.FakeDrafter, @ptrCast(@alignCast(ptr))));
+ }
+
+ fn hold(ptr: *anyopaque, items: []const dr.Hold) anyerror!void {
+ const p = of(ptr);
+ for (items) |it| {
+ try p.at.put(gpa, it.stream, it.position);
+ p.held += it.depth;
+ }
+ return p.inner_hold(ptr, items);
+ }
+
+ /// The running product along the chain, as the DSpark confidence head's survivals are.
+ fn chances(ptr: *anyopaque, s: *sm.Stream, out: []f64) anyerror!bool {
+ const p = of(ptr);
+ const lane = p.head.lanes.getPtr(s) orelse return false;
+ if (out.len > lane.held.items.len) return error.TooFewDrafts;
+ const position = p.at.get(s) orelse return error.NoHold;
+ p.asked += 1;
+ var reach: f64 = 1;
+ for (out, 0..) |*o, j| {
+ const at = position + j;
+ reach *= if ((at *% 2654435761 + j) % 5 == 0) p.unsure else p.sure; // drafted_test's wrong guesses
+ o.* = reach;
+ }
+ return true;
+ }
+
+ fn drafter(p: *Priced) dr.Drafter {
+ const base_ = p.head.drafter();
+ p.vtable = base_.vtable.*;
+ p.inner_hold = base_.vtable.hold;
+ p.vtable.hold = hold;
+ p.vtable.chances = chances;
+ return .{ .ptr = &p.head, .vtable = &p.vtable };
+ }
+
+ fn deinit(p: *Priced) void {
+ p.at.deinit(gpa);
+ p.head.deinit();
+ }
+};
+
+/// Every shared verify's window rows: whether one round gave its streams unequal rows (a ragged verify).
+const Seen = struct {
+ var inner: *const fn (*anyopaque, []const be.Window, []be.Verified) anyerror!void = undefined;
+ var ragged: u32 = 0;
+ var rows: u64 = 0;
+
+ fn verify(ptr: *anyopaque, windows: []const be.Window, out: []be.Verified) anyerror!void {
+ for (windows) |w| {
+ rows += w.rows();
+ if (w.rows() != windows[0].rows()) {
+ ragged += 1;
+ break;
+ }
+ }
+ return inner(ptr, windows, out);
+ }
+};
+
+var costs: [16]Cost = undefined;
+
+fn base() Model {
+ for (&costs, 1..) |*c, w| c.* = .{ .width = @intCast(w), .ms = 5.0 + 0.8 * @as(f64, @floatFromInt(w)) };
+ return .{ .exact_width = 16, .gpu_tokens = true, .window_costs = &costs, .hidden_rows = true, .batch_rows = 32, .max_streams = most_streams };
+}
+
+const Run = struct { tokens: [most_streams][64]u32 = undefined, lens: [most_streams]usize = @splat(0), drafted: u64 = 0, holds: u32 = 0, asked: u32 = 0, ragged: u32 = 0, rows: u64 = 0, held: u64 = 0 };
+
+/// The drafter's Facts.slack for the next runs (0: the depth rule's own; 9: every hold the whole block).
+var slack: u32 = 0;
+
+/// The drafter's chances: equal everywhere, or knowing its wrong guesses (a confidence head that ranks well).
+const Head = enum { flat, knows };
+
+/// Each prompt decoded together (one stream: alone); `floor` null: the chances price rows, else a fixed window.
+fn run(prompts: []const []const u32, drafts: bool, floor: ?f64, head: Head) !Run {
+ var target: fake.Fake = .{ .gpa = gpa };
+ defer target.deinit();
+ var p: Priced = .{ .head = .{ .facts_ = .{ .depth = 9, .step_ms = 0.5, .batched = true, .chances = true, .confidence_floor = floor, .slack = slack } } };
+ if (head == .knows) {
+ p.sure = 0.95;
+ p.unsure = 0.05;
+ }
+ defer p.deinit();
+ var strict: dt.Strict = .{ .inner = target.backend() };
+ defer strict.deinit();
+ var seen = strict.backend();
+ var table = seen.vtable.*;
+ Seen.inner = table.verify;
+ Seen.ragged = 0;
+ Seen.rows = 0;
+ table.verify = Seen.verify;
+ seen.vtable = &table;
+ var wrapped = try Drafted.init(gpa, seen, p.drafter());
+ defer wrapped.deinit();
+ var cfg = try Config.init(gpa, if (drafts) wrapped.facts(base()) else base(), 16, 15);
+ defer cfg.deinit(gpa);
+ var clock: fake.FixedClock = .{};
+ var engine = Engine.init(gpa, &cfg, if (drafts) wrapped.backend() else target.backend(), clock.clock());
+ defer engine.deinit();
+ var streams: [most_streams]sm.Stream = undefined;
+ const n = prompts.len;
+ for (prompts, streams[0..n]) |prompt, *s| s.* = try sm.Stream.init(gpa, .{ .id = "s", .prompt = prompt, .max_new = 48, .eos = &.{96}, .drafts = drafts });
+ defer for (streams[0..n]) |*s| s.deinit(gpa);
+ for (streams[0..n]) |*s| try engine.addStream(s);
+ while (engine.activeCount() > 0) try engine.step();
+ var out: Run = .{ .holds = p.head.holds, .asked = p.asked, .ragged = Seen.ragged, .rows = Seen.rows, .held = p.held };
+ for (streams[0..n], 0..) |*s, i| {
+ out.lens[i] = s.emitted().len;
+ @memcpy(out.tokens[i][0..out.lens[i]], s.emitted());
+ out.drafted += s.drafted;
+ }
+ return out;
+}
+
+fn expectSame(a: Run, b: Run, n: usize) !void {
+ for (0..n) |i| try std.testing.expectEqualSlices(u32, a.tokens[i][0..a.lens[i]], b.tokens[i][0..b.lens[i]]);
+}
+
+/// `n` streams of unequal prompts, each a prefix of drafted_test's long prompt.
+fn unequal(buf: *[most_streams][]const u32, n: usize) []const []const u32 {
+ for (buf[0..n], 0..) |*p, i| p.* = dt.long[0 .. 11 + i];
+ return buf[0..n];
+}
+
+test "a drafter's chances price its chain's rows, and drafted replies equal one-token decode alone and shared" {
+ const sets = [_][]const []const u32{ &.{&dt.p1}, &.{ &dt.p1, &dt.long, &dt.p1 } };
+ for (sets) |ps| {
+ const plain = try run(ps, false, null, .flat);
+ const priced = try run(ps, true, null, .flat);
+ try expectSame(plain, priced, ps.len);
+ try std.testing.expect(priced.asked > 0 and priced.drafted > 0); // the window allocator read the chances
+ const fixed = try run(ps, true, 0, .flat);
+ try expectSame(plain, fixed, ps.len);
+ try std.testing.expect(fixed.drafted > priced.drafted); // a fixed window verifies every draft the rows fit
+ }
+}
+
+test "at 1, 4 and 8 streams the scheduled and the fixed verify equal one-token decode; shared rounds are ragged" {
+ var buf: [most_streams][]const u32 = undefined;
+ for ([_]usize{ 1, 4, 8 }) |n| {
+ const ps = unequal(&buf, n);
+ const plain = try run(ps, false, null, .flat);
+ const scheduled = try run(ps, true, null, .knows);
+ try expectSame(plain, scheduled, n);
+ const fixed = try run(ps, true, 0, .knows);
+ try expectSame(plain, fixed, n);
+ try std.testing.expect(scheduled.asked > 0 and scheduled.drafted > 0);
+ // a confident head's survivals cut each stream's chain where its guess goes wrong: fewer rows verified
+ try std.testing.expect(scheduled.rows < fixed.rows);
+ if (n > 1) try std.testing.expect(scheduled.ragged > 0); // streams' verify lengths differ within a round
+ }
+}
+
+test "the drafter's chances reach the lane facts: rows by chances and holds by grants, or a fixed window" {
+ var p: Priced = .{ .head = .{ .facts_ = .{ .depth = 9, .step_ms = 0.5, .chances = true } } };
+ defer p.deinit();
+ var target: fake.Fake = .{ .gpa = gpa };
+ defer target.deinit();
+ var wrapped: Drafted = .{ .gpa = gpa, .target = target.backend(), .drafter = p.drafter() };
+ defer wrapped.deinit();
+ const m = wrapped.facts(base());
+ try std.testing.expect(m.draft_probabilities and m.granted_depth and m.draft_confidence == null);
+ var cfg = try Config.init(gpa, m, 16, 15);
+ defer cfg.deinit(gpa);
+ try std.testing.expectEqual(@as(i64, 2), cfg.k.draft_slack); // the depth rule's own
+ try std.testing.expect(cfg.k.grant_slack == 1 and cfg.k.short_slack == 0); // a granted width's own
+ p.head.facts_.slack = 1;
+ var one = try Config.init(gpa, wrapped.facts(base()), 16, 15);
+ defer one.deinit(gpa);
+ try std.testing.expectEqual(@as(i64, 1), one.k.draft_slack); // a drafter's own slack past its grant
+ try std.testing.expect(one.k.grant_slack == 1 and one.k.short_slack == 1); // short or not
+ p.head.facts_.confidence_floor = 0;
+ const f = wrapped.facts(base());
+ try std.testing.expect(!f.draft_probabilities and !f.granted_depth and f.draft_confidence.? == 0);
+ p.head.facts_.chances = false;
+ const none = wrapped.facts(base());
+ try std.testing.expect(!none.draft_probabilities and none.draft_confidence == null);
+}
+
+test "a granted depth: the deepest chain first, then the grant plus its slack, and after a grant of none a probe" {
+ const empty: Table = .{};
+ var rule: depth.Rule = .{ .gpa = gpa, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = true, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 };
+ defer rule.deinit();
+ var st: ?depth.State = null;
+ defer if (st) |*s| s.deinit(gpa);
+ const No = struct {
+ fn copy(_: *anyopaque, _: *const depth.Who) bool {
+ return false;
+ }
+ };
+ var unused: u8 = 0;
+ const probe: depth.Probe = .{ .ptr = &unused, .copy = No.copy };
+ var who: depth.Who = .{ .state = &st, .draft_room = 100, .finished = false, .forced = false };
+ try std.testing.expectEqual(@as(i64, 9), try rule.headDepth(&who, null, probe));
+ who.granted = 3;
+ try std.testing.expectEqual(@as(i64, 4), try rule.headDepth(&who, null, probe)); // every held draft granted: one more
+ who.grant_short = true;
+ try std.testing.expectEqual(@as(i64, 3), try rule.headDepth(&who, null, probe)); // drafts left unverified: the grant
+ who.grant_short = false;
+ who.granted = 8;
+ try std.testing.expectEqual(@as(i64, 9), try rule.headDepth(&who, null, probe));
+ rule.k.grant_slack = 2; // a drafter's slack sets both
+ rule.k.short_slack = 2;
+ who.grant_short = true;
+ who.granted = 3;
+ try std.testing.expectEqual(@as(i64, 5), try rule.headDepth(&who, null, probe));
+ who.granted = 0;
+ var drafted: usize = 0;
+ for (0..8) |_| drafted += @intFromBool(try rule.headDepth(&who, null, probe) > 0);
+ try std.testing.expectEqual(@as(usize, 1), drafted); // one probe of draft_slack drafts in the first eight rounds
+}
+
+test "holds as wide as the grants: replies equal one-token decode at 1, 4 and 8 streams, fewer drafts than whole blocks" {
+ var buf: [most_streams][]const u32 = undefined;
+ defer slack = 0;
+ for ([_]usize{ 1, 4, 8 }) |n| {
+ const ps = unequal(&buf, n);
+ const plain = try run(ps, false, null, .flat);
+ for ([_]Head{ .flat, .knows }) |head| {
+ slack = 0;
+ const narrow = try run(ps, true, null, head);
+ slack = 2;
+ const two = try run(ps, true, null, head);
+ slack = 9;
+ const whole = try run(ps, true, null, head);
+ try expectSame(plain, narrow, n);
+ try expectSame(plain, two, n);
+ try expectSame(plain, whole, n);
+ try std.testing.expect(narrow.drafted > 0);
+ try std.testing.expect(narrow.held < two.held and two.held < whole.held);
+ }
+ }
+}
+
+test "hold budgeting queries a stateful copy proposer once for a granted depth" {
+ const empty: Table = .{};
+ var rule: depth.Rule = .{ .gpa = gpa, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 1, .plain_guard = true, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 };
+ defer rule.deinit();
+ var state: ?depth.State = null;
+ defer if (state) |*s| s.deinit(gpa);
+ var whos = [_]depth.Who{.{ .state = &state, .draft_room = 64, .finished = false, .forced = false, .granted = 1 }};
+ const Changing = struct {
+ fn copy(ptr: *anyopaque, _: *const depth.Who) bool {
+ const calls: *usize = @ptrCast(@alignCast(ptr));
+ calls.* += 1;
+ return calls.* > 1;
+ }
+ };
+ var calls: usize = 0;
+ var out: [1]i64 = undefined;
+ try rule.budgets(&whos, .{ .ptr = &calls, .copy = Changing.copy }, false, &out);
+ try std.testing.expectEqual(@as(usize, 1), calls);
+ try std.testing.expectEqual(@as(i64, 2), out[0]);
+}
diff --git a/zig/src/core/lanes/drafted_copy_test.zig b/zig/src/core/lanes/drafted_copy_test.zig
new file mode 100644
index 0000000000..329731f69c
--- /dev/null
+++ b/zig/src/core/lanes/drafted_copy_test.zig
@@ -0,0 +1,135 @@
+//! A drafter whose hold costs a pass of its own: no hold for a stream whose next round is a copy.
+const std = @import("std");
+const Config = @import("config.zig").Config;
+const Engine = @import("engine.zig").Engine;
+const be = @import("backend.zig");
+const sm = @import("stream.zig");
+const fake = @import("fake.zig");
+const dr = @import("drafter.zig");
+const Proposer = @import("proposer.zig").Proposer;
+const Drafted = @import("drafted.zig").Drafted;
+const dt = @import("drafted_test.zig");
+
+const gpa = std.testing.allocator;
+
+/// Copies the fake target's own continuation (it lands whole) whenever the context's length is a multiple of three.
+const Copier = struct {
+ out: [3]u32 = undefined,
+ match: i64 = 0,
+
+ fn propose(ptr: *anyopaque, context: []const u32, most: i64) anyerror![]const u32 {
+ const c: *Copier = @ptrCast(@alignCast(ptr));
+ c.match = 0;
+ if (context.len % 3 != 0 or most < 2) return &.{};
+ var history: [128]u32 = undefined;
+ @memcpy(history[0..context.len], context);
+ const n: usize = @intCast(@min(3, most));
+ for (c.out[0..n], 0..) |*t, k| {
+ t.* = fake.next(history[0 .. context.len + k], null, 0);
+ history[context.len + k] = t.*;
+ }
+ c.match = 100;
+ return c.out[0..n];
+ }
+
+ fn lastMatch(ptr: *anyopaque) i64 {
+ return @as(*Copier, @ptrCast(@alignCast(ptr))).match;
+ }
+
+ const vtable: Proposer.VTable = .{ .propose = propose, .last_match = lastMatch };
+};
+
+/// Each stream's last hold position, and the copy windows verified at a position a hold was asked for.
+const Watch = struct {
+ var inner_hold: *const fn (*anyopaque, []const dr.Hold) anyerror!void = undefined;
+ var inner_verify: *const fn (*anyopaque, []const be.Window, []be.Verified) anyerror!void = undefined;
+ var at: [4]?u64 = @splat(null);
+ var streams: [4]?*sm.Stream = @splat(null);
+ var wasted: u32 = 0;
+ var copies: u32 = 0;
+ var blocks: u32 = 0; // the streams' blocks held, over every hold
+
+ fn slot(s: *sm.Stream) usize {
+ for (&streams, 0..) |*x, i| if (x.* == null or x.* == s) {
+ x.* = s;
+ return i;
+ };
+ unreachable;
+ }
+
+ fn hold(ptr: *anyopaque, items: []const dr.Hold) anyerror!void {
+ blocks += @intCast(items.len);
+ // a joining stream's first hold comes before its first token is read: no copy can be foreseen for it
+ for (items) |it| at[slot(it.stream)] = if (it.stream.emitted().len > 0) it.position else null;
+ return inner_hold(ptr, items);
+ }
+
+ fn verify(ptr: *anyopaque, windows: []const be.Window, out: []be.Verified) anyerror!void {
+ for (windows) |w| if (w.held == 0 and w.tokens.len > 0) {
+ copies += 1;
+ if (at[slot(w.stream)] == w.positions[0]) wasted += 1;
+ };
+ return inner_verify(ptr, windows, out);
+ }
+};
+
+const Run = struct { tokens: [3][64]u32 = undefined, lens: [3]usize = @splat(0), holds: u32 = 0, wasted: u32 = 0, copies: u32 = 0 };
+
+fn run(n: usize, drafts: bool, copy_first: bool) !Run {
+ Watch.at = @splat(null);
+ Watch.streams = @splat(null);
+ Watch.wasted = 0;
+ Watch.copies = 0;
+ Watch.blocks = 0;
+ var target: fake.Fake = .{ .gpa = gpa };
+ defer target.deinit();
+ var head: dt.FakeDrafter = .{ .facts_ = .{ .depth = 4, .step_ms = 0.5, .batched = true, .copy_first = copy_first } };
+ defer head.deinit();
+ var hooked = head.drafter();
+ var table = hooked.vtable.*;
+ Watch.inner_hold = table.hold;
+ table.hold = Watch.hold;
+ hooked.vtable = &table;
+ var wrapped = try Drafted.init(gpa, target.backend(), hooked);
+ defer wrapped.deinit();
+ // the round loop's windows, before the wrapper turns held drafts into host tokens
+ var seen = if (drafts) wrapped.backend() else target.backend();
+ var verified = seen.vtable.*;
+ Watch.inner_verify = verified.verify;
+ verified.verify = Watch.verify;
+ seen.vtable = &verified;
+ const m: @import("config.zig").Model = .{ .exact_width = 16, .gpu_tokens = true, .hidden_rows = true, .batch_rows = 32, .max_streams = 4 };
+ var cfg = try Config.init(gpa, if (drafts) wrapped.facts(m) else m, 16, 15);
+ defer cfg.deinit(gpa);
+ var clock: fake.FixedClock = .{};
+ var engine = Engine.init(gpa, &cfg, seen, clock.clock());
+ defer engine.deinit();
+ var copiers: [3]Copier = @splat(.{});
+ var streams: [3]sm.Stream = undefined;
+ for (streams[0..n], copiers[0..n], 0..) |*s, *c, i| s.* = try sm.Stream.init(gpa, .{ .id = "s", .prompt = dt.long[0 .. 12 + i], .max_new = 40, .eos = &.{96}, .drafts = drafts, .proposer = .{ .ptr = c, .vtable = &Copier.vtable } });
+ defer for (streams[0..n]) |*s| s.deinit(gpa);
+ for (streams[0..n]) |*s| try engine.addStream(s);
+ while (engine.activeCount() > 0) try engine.step();
+ var out: Run = .{ .holds = Watch.blocks, .wasted = Watch.wasted, .copies = Watch.copies };
+ for (streams[0..n], 0..) |*s, i| {
+ out.lens[i] = s.emitted().len;
+ @memcpy(out.tokens[i][0..out.lens[i]], s.emitted());
+ }
+ return out;
+}
+
+test "a copy ahead takes the round: with copy_first no hold is asked for it, and replies equal one-token decode" {
+ for ([_]usize{ 1, 3 }) |n| {
+ const plain = try run(n, false, false);
+ const before = try run(n, true, false);
+ const after = try run(n, true, true);
+ for (0..n) |i| {
+ try std.testing.expectEqualSlices(u32, plain.tokens[i][0..plain.lens[i]], before.tokens[i][0..before.lens[i]]);
+ try std.testing.expectEqualSlices(u32, plain.tokens[i][0..plain.lens[i]], after.tokens[i][0..after.lens[i]]);
+ }
+ try std.testing.expect(before.copies > 0 and after.copies > 0);
+ try std.testing.expect(before.wasted > 0); // a hold of one draft beside each copy, dropped by the copy round
+ try std.testing.expectEqual(@as(u32, 0), after.wasted);
+ try std.testing.expect(after.holds < before.holds);
+ }
+}
diff --git a/zig/src/core/lanes/drafter.zig b/zig/src/core/lanes/drafter.zig
index 2d82d40d04..8ecd22db47 100644
--- a/zig/src/core/lanes/drafter.zig
+++ b/zig/src/core/lanes/drafter.zig
@@ -18,6 +18,11 @@ pub const Facts = struct {
window_costs: []const Cost = &.{}, // the target's window ms by rows (timed at load), for the depth rule
lane_costs: []const Cost = &.{}, // the same for the lane planner (Model.lane_costs)
tree_nodes: u32 = 0, // hold() without lanes grows its own tree of up to `depth` nodes, at most this many (tree, chances)
+ chances: bool = false, // chances() gives each held chain draft's chance: verify rows go by them (Model.granted_depth)
+ confidence_floor: ?f64 = null, // with chances: every round drafts the deepest chain and verifies drafts this likely
+ slack: u32 = 0, // with chances: drafts a hold offers past the stream's last grant (0: the depth rule's own)
+ copy_first: bool = false, // a hold costs a pass of its own: none is asked for a stream with a copy ahead
+ hold_costs: []const @import("hold_price.zig").HoldCost = &.{}, // that pass's ms by blocks and depth (timed at load)
};
/// Target rows [start, start + features.rows) and the token after each; `rows`, when given, picks which to absorb.
diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig
index 5d357c646d..3bf6c297da 100644
--- a/zig/src/core/lanes/engine.zig
+++ b/zig/src/core/lanes/engine.zig
@@ -16,6 +16,7 @@ const lone_costs = @import("lone_costs.zig");
const open = @import("open.zig");
const handover = @import("handover.zig");
const trim = @import("trim.zig");
+const pipelined = @import("pipelined.zig");
const Stream = sm.Stream;
const Feed = be.Feed;
const LogRow = @import("logprob.zig").Row;
@@ -24,7 +25,7 @@ const f = ev.f;
const str = trail.str;
const int = trail.int;
const Outcome = struct { got: []const u32, path: []const u32, cut: ?usize, follow: []const u32 };
-const Result = struct { rows: u32, keep: u32, got: []const u32 };
+pub const Result = struct { rows: u32, keep: u32, got: []const u32 };
const Unread = struct { event: usize, handle: u64 };
pub const Engine = struct {
const pieces_ = @import("pieces.zig");
@@ -70,6 +71,7 @@ pub const Engine = struct {
.draft_step_ms = cfg.draft_step_ms,
.plain_guard = cfg.plain_guard,
.node_probabilities = cfg.node_probabilities,
+ .granted_depth = cfg.granted_depth,
.confidence_floor = cfg.draft_confidence,
.batch_rows = cfg.batch_rows,
.batched_heads = cfg.draft_streams,
@@ -124,7 +126,7 @@ pub const Engine = struct {
if (live.items.len > 1 and e.cfg.family_streams) {
// a shared round is synchronous: a stream that ran a step ahead lands its queued token first
for (live.items) |s| {
- if (s.inflight != null) try e.landInflight(s);
+ if (s.inflight != null) try pipelined.land(e, s);
}
var kept: usize = 0;
for (live.items) |s| {
@@ -147,7 +149,7 @@ pub const Engine = struct {
for (live.items) |s| {
// logprobs and grammars take verify windows: rows' logits are read, or bits made, before a forward
const windowed = (e.cfg.family_mtp and s.drafts) or !e.cfg.pipelined or s.logprobs != null or s.grammar != null;
- const r = if (windowed) try e.familyRound(s, null) else try e.pipelinedRound(s);
+ const r = if (windowed) try e.familyRound(s, null) else try pipelined.round(e, s);
s.min_rows = if (s.min_rows == 0) r.rows else @min(s.min_rows, r.rows);
}
}
@@ -195,7 +197,7 @@ pub const Engine = struct {
}
/// One stream's round: verify pending and drafted tokens together, keep drafts to the first mismatch.
- fn familyRound(e: *Engine, s: *Stream, copied: ?win.Copy) !Result {
+ pub fn familyRound(e: *Engine, s: *Stream, copied: ?win.Copy) !Result {
e.clock.start();
const priced = s.priced;
s.priced = null;
@@ -490,64 +492,6 @@ pub const Engine = struct {
}
}
- /// One token, the next forward queued first; a copied continuation ahead switches to verify windows.
- fn pipelinedRound(e: *Engine, s: *Stream) !Result {
- const before = s.mode;
- // an adopted stream holds neither a mode nor a queued step: it starts where a drain step leaves one
- const mode: sm.Mode = s.mode orelse if (s.inflight == null) .verify else .pipe;
- var result: Result = .{ .rows = 1, .keep = 1, .got = &.{} };
- if (mode == .verify or mode == .exit) {
- const verifying = mode == .verify and e.cfg.family_width >= 2;
- const proposal: win.Copy = if (verifying) try win.copyProposal(e, s) else .{};
- if (proposal.tokens.len > 0) {
- const copied = try e.arena.allocator().dupe(u32, proposal.tokens);
- result = try e.familyRound(s, .{ .tokens = copied, .recalled = proposal.recalled });
- if (result.keep == 1 or s.force.items.len > 0) s.mode = .exit;
- } else {
- // nothing to copy: back to steps queued ahead (this one lands next round)
- s.mode = .pipe;
- try e.queueNext(s, .{ .value = s.pending.? });
- }
- try trail.pipe(e, s, before, result.got);
- return result;
- }
- var current: Feed = .{ .handle = s.inflight.? };
- s.inflight = null;
- if (try e.forcedNext(s)) |t| current = .{ .value = t };
- var token: u32 = undefined;
- if (mode == .drain) {
- token = try e.readFeed(current); // the last queued step: no new one
- s.mode = .verify;
- } else {
- try e.queueNext(s, current); // the GPU starts the next step first
- token = try e.readFeed(current);
- }
- s.rounds += 1;
- const landed = try s.commit(e.gpa, &.{token}, &.{});
- s.pending = token;
- if (s.recall) |r| r.noteHead(1); // a step lands one token: the bar a copy must beat
- if (e.cfg.root_copies) s.root_copy_miss = false;
- const piping = e.cfg.family_width >= 2 and (s.mode orelse .pipe) == .pipe and !s.finished and s.drafts;
- if (piping and (try win.copyProposal(e, s)).tokens.len > 0)
- s.mode = .drain; // a copy window is ahead: land the queued step
- result.got = try e.arena.allocator().dupe(u32, s.emitted()[s.emitted().len - landed ..]);
- try trail.pipe(e, s, before, result.got);
- return result;
- }
-
- /// Land the queued token without queuing another (a shared round is synchronous).
- fn landInflight(e: *Engine, s: *Stream) !void {
- var current: Feed = .{ .handle = s.inflight.? };
- s.inflight = null;
- if (try e.forcedNext(s)) |t| current = .{ .value = t };
- const token = try e.readFeed(current);
- s.rounds += 1;
- const landed = try s.commit(e.gpa, &.{token}, &.{});
- s.pending = token;
- s.mode = .verify;
- try trail.event(e, &.{ f("ev", str("land")), f("stream", str(s.id)), f("got", .{ .u32s = s.emitted()[s.emitted().len - landed ..] }) });
- }
-
/// The first token's draw: among the grammar's allowed tokens when the stream has one that constrains it.
pub fn drawFirst(e: *Engine, s: *Stream, position: u64) !u64 {
const g = s.grammar orelse return e.backend.first(s, position);
diff --git a/zig/src/core/lanes/hold_price.zig b/zig/src/core/lanes/hold_price.zig
new file mode 100644
index 0000000000..c3eeb0208a
--- /dev/null
+++ b/zig/src/core/lanes/hold_price.zig
@@ -0,0 +1,84 @@
+//! A drafter's hold pass priced from its timed passes: by the blocks it holds and their depth.
+const std = @import("std");
+
+/// A drafter's hold pass of `streams` blocks, each `depth` drafts deep, in ms (timed at load).
+pub const HoldCost = struct { streams: u32, depth: u32, ms: f64 };
+
+/// A hold pass's ms: linear in depth at each timed stream count, then in streams (past the widest, by its slope).
+pub fn holdMs(costs: []const HoldCost, streams: usize, depth: i64) f64 {
+ var below: ?u32 = null;
+ var above: ?u32 = null;
+ for (costs) |c| {
+ if (c.streams <= streams and (below == null or c.streams > below.?)) below = c.streams;
+ if (c.streams >= streams and (above == null or c.streams < above.?)) above = c.streams;
+ }
+ const lo = below orelse above.?;
+ const hi = above orelse lo;
+ const at_lo = atDepth(costs, lo, depth);
+ if (hi == lo) {
+ if (above != null) return at_lo;
+ // past the widest timed count: the slope from the count below it
+ var under: ?u32 = null;
+ for (costs) |c| if (c.streams < lo and (under == null or c.streams > under.?)) {
+ under = c.streams;
+ };
+ const u = under orelse return at_lo * @as(f64, @floatFromInt(streams)) / @as(f64, @floatFromInt(lo));
+ const slope = (at_lo - atDepth(costs, u, depth)) / @as(f64, @floatFromInt(lo - u));
+ return at_lo + slope * @as(f64, @floatFromInt(streams - lo));
+ }
+ const t = @as(f64, @floatFromInt(streams - lo)) / @as(f64, @floatFromInt(hi - lo));
+ return at_lo + t * (atDepth(costs, hi, depth) - at_lo);
+}
+
+/// Interpolate neighboring depths, or extrapolate the nearest end segment (one alone: as timed).
+fn atDepth(costs: []const HoldCost, streams: u32, depth: i64) f64 {
+ var a: ?HoldCost = null;
+ var b: ?HoldCost = null;
+ for (costs) |c| {
+ if (c.streams != streams) continue;
+ if (c.depth == depth) return c.ms;
+ if (c.depth < depth and (a == null or c.depth > a.?.depth)) a = c;
+ if (c.depth > depth and (b == null or c.depth < b.?.depth)) b = c;
+ }
+ if (a == null) {
+ a = b.?;
+ b = null;
+ for (costs) |c| if (c.streams == streams and c.depth > a.?.depth and (b == null or c.depth < b.?.depth)) {
+ b = c;
+ };
+ if (b == null) return a.?.ms;
+ } else if (b == null) {
+ b = a.?;
+ a = null;
+ for (costs) |c| if (c.streams == streams and c.depth < b.?.depth and (a == null or c.depth > a.?.depth)) {
+ a = c;
+ };
+ if (a == null) return b.?.ms;
+ }
+ const x = a.?;
+ const y = b.?;
+ const t = (@as(f64, @floatFromInt(depth)) - @as(f64, @floatFromInt(x.depth))) / @as(f64, @floatFromInt(y.depth - x.depth));
+ return x.ms + t * (y.ms - x.ms);
+}
+
+test "hold prices retain measured kernel transitions and use the adjacent depth segment" {
+ const t = [_]HoldCost{
+ .{ .streams = 8, .depth = 4, .ms = 18 },
+ .{ .streams = 8, .depth = 1, .ms = 10 },
+ .{ .streams = 8, .depth = 2, .ms = 17 },
+ };
+ try std.testing.expectEqual(@as(f64, 17), holdMs(&t, 8, 2));
+ try std.testing.expectEqual(@as(f64, 17.5), holdMs(&t, 8, 3));
+ try std.testing.expectEqual(@as(f64, 18.5), holdMs(&t, 8, 5));
+ try std.testing.expectEqual(@as(f64, 3), holdMs(&t, 8, 0));
+}
+
+test "a hold's ms between and past the timed passes" {
+ const t = [_]HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1.0 }, .{ .streams = 1, .depth = 4, .ms = 2.5 }, .{ .streams = 4, .depth = 1, .ms = 3.0 }, .{ .streams = 4, .depth = 4, .ms = 4.5 }, .{ .streams = 8, .depth = 1, .ms = 4.0 } };
+ try std.testing.expectApproxEqAbs(@as(f64, 1.0), holdMs(&t, 1, 1), 1e-12);
+ try std.testing.expectApproxEqAbs(@as(f64, 2.0), holdMs(&t, 1, 3), 1e-12); // linear in depth
+ try std.testing.expectApproxEqAbs(@as(f64, 3.0), holdMs(&t, 1, 5), 1e-12); // and past the deepest
+ try std.testing.expectApproxEqAbs(@as(f64, 1.0 + 2.0 / 3.0), holdMs(&t, 2, 1), 1e-12); // linear in streams
+ try std.testing.expectApproxEqAbs(@as(f64, 4.0), holdMs(&t, 8, 3), 1e-12); // one depth timed: as timed
+ try std.testing.expectApproxEqAbs(@as(f64, 5.0), holdMs(&t, 12, 1), 1e-12); // past the widest: the slope below it
+}
diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig
index 280b16ef10..cc0271d636 100644
--- a/zig/src/core/lanes/lanes.zig
+++ b/zig/src/core/lanes/lanes.zig
@@ -18,6 +18,7 @@ pub const cost_rule = @import("cost_rule.zig");
pub const gpu_rule = @import("gpu_rule.zig");
pub const gpu_full = @import("gpu_full.zig");
pub const depth = @import("depth.zig");
+pub const hold_price = @import("hold_price.zig");
pub const mtp_policy = @import("mtp_policy.zig");
pub const trim = @import("trim.zig");
pub const config = @import("config.zig");
@@ -57,6 +58,8 @@ test {
_ = @import("recorder_test.zig");
_ = @import("gpu_full_test.zig");
_ = @import("drafted_test.zig");
+ _ = @import("drafted_chances_test.zig");
+ _ = @import("drafted_copy_test.zig");
_ = @import("root_copy_test.zig");
_ = @import("drafted_grammar_test.zig");
_ = @import("windows_grammar_test.zig");
diff --git a/zig/src/core/lanes/pipelined.zig b/zig/src/core/lanes/pipelined.zig
new file mode 100644
index 0000000000..99de33304b
--- /dev/null
+++ b/zig/src/core/lanes/pipelined.zig
@@ -0,0 +1,70 @@
+//! A lone stream's pipelined steps: the next forward queued before the last one's token is read.
+const be = @import("backend.zig");
+const sm = @import("stream.zig");
+const ev = @import("events.zig");
+const win = @import("windows.zig");
+const trail = @import("trail.zig");
+const Engine = @import("engine.zig").Engine;
+const Result = @import("engine.zig").Result;
+const Stream = sm.Stream;
+const Feed = be.Feed;
+const f = ev.f;
+const str = trail.str;
+
+/// One token, the next forward queued first; a copied continuation ahead switches to verify windows.
+pub fn round(e: *Engine, s: *Stream) !Result {
+ const before = s.mode;
+ // an adopted stream holds neither a mode nor a queued step: it starts where a drain step leaves one
+ const mode: sm.Mode = s.mode orelse if (s.inflight == null) .verify else .pipe;
+ var result: Result = .{ .rows = 1, .keep = 1, .got = &.{} };
+ if (mode == .verify or mode == .exit) {
+ const verifying = mode == .verify and e.cfg.family_width >= 2;
+ const proposal: win.Copy = if (verifying) try win.copyProposal(e, s) else .{};
+ if (proposal.tokens.len > 0) {
+ const copied = try e.arena.allocator().dupe(u32, proposal.tokens);
+ result = try e.familyRound(s, .{ .tokens = copied, .recalled = proposal.recalled });
+ if (result.keep == 1 or s.force.items.len > 0) s.mode = .exit;
+ } else {
+ // nothing to copy: back to steps queued ahead (this one lands next round)
+ s.mode = .pipe;
+ try e.queueNext(s, .{ .value = s.pending.? });
+ }
+ try trail.pipe(e, s, before, result.got);
+ return result;
+ }
+ var current: Feed = .{ .handle = s.inflight.? };
+ s.inflight = null;
+ if (try e.forcedNext(s)) |t| current = .{ .value = t };
+ var token: u32 = undefined;
+ if (mode == .drain) {
+ token = try e.readFeed(current); // the last queued step: no new one
+ s.mode = .verify;
+ } else {
+ try e.queueNext(s, current); // the GPU starts the next step first
+ token = try e.readFeed(current);
+ }
+ s.rounds += 1;
+ const landed = try s.commit(e.gpa, &.{token}, &.{});
+ s.pending = token;
+ if (s.recall) |r| r.noteHead(1); // a step lands one token: the bar a copy must beat
+ if (e.cfg.root_copies) s.root_copy_miss = false;
+ const piping = e.cfg.family_width >= 2 and (s.mode orelse .pipe) == .pipe and !s.finished and s.drafts;
+ if (piping and (try win.copyProposal(e, s)).tokens.len > 0)
+ s.mode = .drain; // a copy window is ahead: land the queued step
+ result.got = try e.arena.allocator().dupe(u32, s.emitted()[s.emitted().len - landed ..]);
+ try trail.pipe(e, s, before, result.got);
+ return result;
+}
+
+/// Land the queued token without queuing another (a shared round is synchronous).
+pub fn land(e: *Engine, s: *Stream) !void {
+ var current: Feed = .{ .handle = s.inflight.? };
+ s.inflight = null;
+ if (try e.forcedNext(s)) |t| current = .{ .value = t };
+ const token = try e.readFeed(current);
+ s.rounds += 1;
+ const landed = try s.commit(e.gpa, &.{token}, &.{});
+ s.pending = token;
+ s.mode = .verify;
+ try trail.event(e, &.{ f("ev", str("land")), f("stream", str(s.id)), f("got", .{ .u32s = s.emitted()[s.emitted().len - landed ..] }) });
+}
diff --git a/zig/src/core/lanes/stream.zig b/zig/src/core/lanes/stream.zig
index c3e22e7869..ff85c8a3e7 100644
--- a/zig/src/core/lanes/stream.zig
+++ b/zig/src/core/lanes/stream.zig
@@ -234,6 +234,8 @@ pub const Stream = struct {
served: ?u64 = null,
proposal_depth: ?u32 = null, // null: legacy; otherwise the last routed request, zero until it succeeds
granted: ?u32 = null,
+ grant_short: bool = false, // the last grant verified fewer drafts than the stream held
+ grant_gain: ?f64 = null, // the drafts its last grant expected to keep (the chances it was priced by)
copy_width: ?u32 = null,
pub fn init(gpa: Allocator, spec: Spec) !Stream {
@@ -302,6 +304,8 @@ pub const Stream = struct {
s.inflight = null;
s.served = null;
s.granted = null;
+ s.grant_short = false;
+ s.grant_gain = null;
s.copy_width = null;
s.proposal_depth = null;
}
diff --git a/zig/src/core/lanes/windows.zig b/zig/src/core/lanes/windows.zig
index fd7ffec95d..c9e8e8ea2b 100644
--- a/zig/src/core/lanes/windows.zig
+++ b/zig/src/core/lanes/windows.zig
@@ -226,7 +226,7 @@ pub fn probe(e: *Engine) depth.Probe {
}
pub fn who(s: *Stream) depth.Who {
- return .{ .state = &s.depth, .draft_room = s.draftRoom(), .finished = s.finished, .forced = s.force.items.len > 0, .granted = s.granted, .stream = s, .sampled = s.sampling != null };
+ return .{ .state = &s.depth, .draft_room = s.draftRoom(), .finished = s.finished, .forced = s.force.items.len > 0, .granted = s.granted, .grant_short = s.grant_short, .gain = s.grant_gain, .stream = s, .sampled = s.sampling != null };
}
/// Whether any of a round's streams draws keyed samples (its rows carry their host work).
@@ -282,8 +282,16 @@ pub fn allocate(e: *Engine, plans: []Plan) !void {
const ms = own.get(1 + @as(i64, counts[0]));
try trail.event(e, &.{ f("ev", trail.str("lone-price")), f("stream", trail.str(plans[0].stream.id)), f("rows", trail.int(counts[0] + 1)), f("expected", ev.bits(expected)), f("ms", if (ms) |value| ev.bits(value) else .null), f("probe", .{ .bool = trial != null }) });
}
- for (plans, counts) |*p, c| {
- if (p.kind == .head) p.stream.granted = c;
+ for (plans, counts, probs) |*p, c, pr| {
+ if (p.kind == .head) {
+ p.stream.granted = c;
+ if (e.cfg.granted_depth) {
+ p.stream.grant_short = c < p.count();
+ var gain: f64 = 0;
+ for (pr[0..@min(c, pr.len)]) |chance| gain += chance;
+ p.stream.grant_gain = gain;
+ }
+ }
if (p.kind == .forced or p.count() == 0 or c == p.count()) continue;
try cut.prefix(a, p, c);
}
@@ -393,7 +401,7 @@ fn planChances(e: *Engine, p: Plan, n: usize) ![]const f64 {
if (!p.context_copy or p.branch_from == 0 or p.parents == null or n < p.branch_from) return nodeChances(e, p.stream, n);
const drafts = p.branch_from - 1;
const out = try e.arena.allocator().alloc(f64, n);
- if (drafts > 0) @memcpy(out[0..drafts], try nodeChances(e, p.stream, drafts));
+ if (drafts > 0) try nodeChancesInto(e, p.stream, out[0..drafts]);
const rate = copyRate(e, p.stream, p.copy_trusted);
for (out[drafts..], p.parents.?[drafts..n]) |*o, parent| o.* = rate * (if (parent >= 0) out[@intCast(parent)] else 1);
return out;
@@ -460,32 +468,36 @@ test "a copy is trusted after a long match, or out of the prompt early in the re
/// The head's chance for each held draft, else chain chances from the stream's per-depth acceptance.
pub fn nodeChances(e: *Engine, s: *Stream, n: usize) ![]const f64 {
- const a = e.arena.allocator();
+ const out = try e.arena.allocator().alloc(f64, n);
+ try nodeChancesInto(e, s, out);
+ return out;
+}
+
+fn nodeChancesInto(e: *Engine, s: *Stream, out: []f64) !void {
if (e.cfg.node_probabilities) {
if (e.backend.vtable.probabilities) |chances| {
- const out = try a.alloc(f64, n);
- if (try chances(e.backend.ptr, s, out)) return out;
+ if (try chances(e.backend.ptr, s, out)) return;
}
}
if (e.cfg.draft_confidence != null) return error.NoDraftProbabilities;
- return alloc.chainProbabilities(a, try e.rule.rates(who(s)), n);
+ alloc.chainProbabilitiesInto(try e.rule.rates(who(s)), out);
}
/// Structured output: the rows an accepted path can use, each row's bits, and the grammar's drafts after a lone chain.
pub fn constrain(e: *Engine, p: *Plan) !void {
const g = p.stream.grammar orelse return;
const a = e.arena.allocator();
- if (p.held > 0) {
- // the held drafts (a chain or a tree, in window order) come back as host tokens, grafts after them
+ const held_count = p.held;
+ const rows = try a.alloc(u32, 1 + held_count + p.tokens.len);
+ rows[0] = p.stream.pending.?;
+ @memcpy(rows[1 + held_count ..], p.tokens);
+ if (held_count > 0) {
+ // Read held drafts directly into their final rows, followed by any grafts.
const read = e.backend.vtable.held orelse return error.HeldDraftsUnreadable;
- const held = try a.alloc(u32, p.held);
- try read(e.backend.ptr, p.stream, held);
- p.tokens = try std.mem.concat(a, u32, &.{ held, p.tokens });
+ try read(e.backend.ptr, p.stream, rows[1 .. 1 + held_count]);
+ p.tokens = rows[1..];
p.held = 0;
}
- const rows = try a.alloc(u32, 1 + p.tokens.len);
- rows[0] = p.stream.pending.?;
- @memcpy(rows[1..], p.tokens);
const grows = p.stream.drafts and p.kind != .forced and e.alone;
const room = if (grows) @max(rows.len, e.cfg.family_width) else rows.len;
const w = try g.window(a, rows, try accept.rowParents(a, rows.len, p.parents), p.kind == .forced, room);
@@ -540,3 +552,7 @@ pub fn tokens(e: *Engine, w: be.Window, o: be.Verified) ![]u32 {
@memcpy(out[1 + w.held ..], w.tokens);
return out;
}
+
+test "declined and grafted probabilities need only their final round buffer" {
+ try @import("windows_memory_test.zig").check(nodeChances, planChances);
+}
diff --git a/zig/src/core/lanes/windows_grammar_test.zig b/zig/src/core/lanes/windows_grammar_test.zig
index fee374c63b..c90ce91b61 100644
--- a/zig/src/core/lanes/windows_grammar_test.zig
+++ b/zig/src/core/lanes/windows_grammar_test.zig
@@ -93,3 +93,38 @@ test "a tree shaped as a chain takes the grammar's own drafts as chain rows, and
try std.testing.expectEqual(@as(usize, 0), g.at);
}
}
+
+test "held drafts and grafts keep row order through grammar trimming" {
+ var cfg = try et.model();
+ defer cfg.deinit(gpa);
+ var target: fake.Fake = .{ .gpa = gpa };
+ defer target.deinit();
+ var backend = target.backend();
+ var vtable = backend.vtable.*;
+ vtable.held = struct {
+ fn read(_: *anyopaque, _: *sm.Stream, out: []u32) !void {
+ try std.testing.expectEqual(@as(usize, 2), out.len);
+ @memcpy(out, &[_]u32{ 5, 7 });
+ }
+ }.read;
+ backend.vtable = &vtable;
+ var clock: fake.FixedClock = .{};
+ var engine = Engine.init(gpa, &cfg, backend, clock.clock());
+ defer engine.deinit();
+ for ([_]u32{ 9, 4 }) |graft| {
+ var g: Spell = .{ .want = &.{ 5, 7, 9 } };
+ var k: grammar.Constraint = .init(g.rules(), (fake.vocab + 31) / 32, null);
+ var s = try sm.Stream.init(gpa, .{ .id = "held", .prompt = &.{ 1, 2 }, .max_new = 8, .drafts = false, .grammar = &k });
+ defer s.deinit(gpa);
+ s.pending = 3;
+ const tokens = [_]u32{graft};
+ var p: win.Plan = .{ .stream = &s, .position = 2, .kind = .head, .held = 2, .tokens = &tokens };
+ try win.constrain(&engine, &p);
+ try std.testing.expectEqual(@as(u32, 0), p.held);
+ const expected: []const u32 = &.{ 5, 7, 9 };
+ try std.testing.expectEqual(@as(u32, if (graft == 9) 0 else 3), p.grammar_from);
+ try std.testing.expectEqualSlices(u32, expected, p.tokens);
+ const w = try win.build(&engine, p, false);
+ try std.testing.expectEqualSlices(u32, expected, w.tokens);
+ }
+}
diff --git a/zig/src/core/lanes/windows_memory_test.zig b/zig/src/core/lanes/windows_memory_test.zig
new file mode 100644
index 0000000000..db08abf447
--- /dev/null
+++ b/zig/src/core/lanes/windows_memory_test.zig
@@ -0,0 +1,50 @@
+//! Probability planning must fit its final output into a bounded round arena.
+const std = @import("std");
+const Engine = @import("engine.zig").Engine;
+const Stream = @import("stream.zig").Stream;
+const win = @import("windows.zig");
+
+pub fn check(comptime node: anytype, comptime hybrid: anytype) !void {
+ const a = std.testing.allocator;
+ var cfg = try @import("config.zig").Config.init(a, .{ .exact_width = 16, .drafts = 3, .mtp = true, .speculate = true, .draft_probabilities = true, .draft_prior = &.{ 0.8, 0.5 } }, 16, 3);
+ defer cfg.deinit(a);
+ var target: @import("fake.zig").Fake = .{ .gpa = a };
+ defer target.deinit();
+ var clock: @import("fake.zig").FixedClock = .{};
+ var backend = target.backend();
+ var vtable = backend.vtable.*;
+ vtable.probabilities = struct {
+ fn decline(_: *anyopaque, _: *Stream, out: []f64) !bool {
+ @memset(out, 99);
+ return false;
+ }
+ }.decline;
+ backend.vtable = &vtable;
+ var limited = std.testing.FailingAllocator.init(a, .{});
+ var e = Engine.init(a, &cfg, backend, clock.clock());
+ defer e.deinit();
+ var s = try Stream.init(a, .{ .id = "bounded", .prompt = &.{ 1, 2 }, .max_new = 64 });
+ defer s.deinit(a);
+ _ = try e.rule.rates(win.who(&s));
+ // The round gets only enough arena space for its final probability buffer.
+ e.arena.deinit();
+ e.arena = std.heap.ArenaAllocator.init(limited.allocator());
+ _ = try e.arena.allocator().alloc(f64, 32);
+ const capacity = e.arena.queryCapacity();
+ limited.fail_index = limited.alloc_index;
+ limited.resize_fail_index = limited.resize_index;
+ for (0..2) |mode| {
+ try std.testing.expect(e.arena.reset(.retain_capacity));
+ _ = try e.arena.allocator().alloc(u8, capacity - 16 * @sizeOf(f64));
+ var parents: [16]i32 = undefined;
+ @memset(&parents, 0);
+ parents[0] = -1;
+ const p: win.Plan = .{ .stream = &s, .position = 2, .kind = .head, .tokens = &.{}, .parents = &parents, .branch_from = 4, .context_copy = true, .copy_trusted = false };
+ const out = if (mode == 0) try node(&e, &s, 16) else try hybrid(&e, p, 16);
+ try std.testing.expectApproxEqAbs(@as(f64, 0.8), out[0], 1e-12);
+ try std.testing.expectApproxEqAbs(@as(f64, 0.4), out[1], 1e-12);
+ try std.testing.expectApproxEqAbs(@as(f64, 0.2), out[2], 1e-12);
+ if (mode == 1) try std.testing.expectApproxEqAbs(@as(f64, 0.064), out[3], 1e-12);
+ }
+ try std.testing.expect(!limited.has_induced_failure);
+}
diff --git a/zig/src/core/tile_qmm.zig b/zig/src/core/tile_qmm.zig
index 030bd097e1..dbc0da8b68 100644
--- a/zig/src/core/tile_qmm.zig
+++ b/zig/src/core/tile_qmm.zig
@@ -167,10 +167,26 @@ pub fn encodeTwin(
y: mtl.Buffer,
y_off: usize,
rows: usize,
+) !void {
+ // the window kernel only for matrices of window_n outputs or more
+ return encodeTwinFrom(e, p, m, x, x_off, y, y_off, rows, if (m.n >= window_n) window_rows else twin_rows + 1);
+}
+
+/// encodeTwin with the window kernel from `from` rows on (the scalar twin below them), for a caller's measured cut.
+pub fn encodeTwinFrom(
+ e: mtl.ComputeEncoder,
+ p: *const Pipelines,
+ m: Matrix,
+ x: mtl.Buffer,
+ x_off: usize,
+ y: mtl.Buffer,
+ y_off: usize,
+ rows: usize,
+ from: usize,
) !void {
const i = try width(m);
if (rows == 0 or rows > twin_rows) return error.NoRows;
- const window = rows >= window_rows and m.n >= window_n;
+ const window = rows >= from;
e.setPipeline(if (window) p.rows[i] else if (rows > 1) p.twin_few[i] else p.twin[i]);
bind(e, m, .{ .k = @intCast(m.k), .n = @intCast(m.n), .r = @intCast(rows) }, x, x_off, y, y_off);
if (window) return e.dispatchGroups(.{ .width = m.n / window_columns }, .{ .width = 64 });
diff --git a/zig/src/families/lfm2_moe/backend.zig b/zig/src/families/lfm2_moe/backend.zig
index d713c9ec24..2e5130db84 100644
--- a/zig/src/families/lfm2_moe/backend.zig
+++ b/zig/src/families/lfm2_moe/backend.zig
@@ -10,6 +10,14 @@ const snapshot = @import("snapshot.zig");
const Model = @import("model.zig").Model;
const be = lanes.backend;
+/// Where a drafter holds a stream's drafts on the device, for verify windows that take them there (Window.held).
+pub const HeldSource = struct {
+ ptr: *anyopaque,
+ get: *const fn (ptr: *anyopaque, s: *lanes.Stream) ?Device,
+ host: *const fn (ptr: *anyopaque, s: *lanes.Stream, out: []u32) anyerror!void, // where `get` gives none
+};
+pub const Device = struct { src: mtl.Buffer, off: usize, stride: u32, wait: fwd.Wait };
+
pub const Options = struct { capacity: usize, chunk: usize = 128, streams: u32 = 8 };
pub const Metal = struct {
@@ -23,6 +31,13 @@ pub const Metal = struct {
/// Forward ms by window width, timed at load: the lane core weighs drafts against them.
costs: [st.window_rows]lanes.config.Cost = undefined,
timed: usize = 0,
+ /// Shared-row increments past the qualified window, anchored at that window's measured cost.
+ shared: [st.batch_rows - st.window_rows]lanes.config.Cost = undefined,
+ shared_timed: usize = 0,
+ /// Forwards run so far: a stream's tapped rows in the scratch are current while their gen matches.
+ gen: u64 = 0,
+ /// A drafter whose drafts verify windows read on the device (null: windows carry host tokens only).
+ held_source: ?HeldSource = null,
pub fn init(gpa: std.mem.Allocator, model: *Model, options: Options) !*Metal {
const out = try gpa.create(Metal);
@@ -54,7 +69,9 @@ pub const Metal = struct {
}
pub fn facts(self: *const Metal) lanes.Model {
- return factsFor(self.width, self.options.streams, self.costs[0..self.timed]);
+ var out = factsFor(self.width, self.options.streams, self.costs[0..self.timed]);
+ out.shared_costs = self.shared[0..self.shared_timed];
+ return out;
}
/// Each window width's forward ms on a 256-token history (the fastest of three, after the GPU's clocks ramp up).
@@ -84,11 +101,37 @@ pub const Metal = struct {
cost.* = .{ .width = @intCast(w), .ms = best };
}
std.log.info("LFM2 window ms by width: 1 {d:.1}, 2 {d:.1}, 4 {d:.1}, 8 {d:.1}, 16 {d:.1}", .{ self.costs[0].ms, self.costs[1].ms, self.costs[3].ms, self.costs[7].ms, self.costs[15].ms });
+ // Two qualified windows measure shared-row increments against the same shared shape at 16 rows.
+ var other = try st.Cache.init(self.gpa, self.model.device, &self.model.config.attention, 512);
+ defer other.deinit();
+ try fwd.run(self.model, &self.scratch, &.{.{ .cache = &other, .rows = 256 }}, &ids, false, .last);
+ var shared_base: f64 = 0;
+ for (st.window_rows..st.batch_rows + 1) |rows| {
+ const first = @min(st.window_rows, rows - 1);
+ var best: f64 = std.math.floatMax(f64);
+ for (0..4) |trial| {
+ var t0: platform.c.timespec = undefined;
+ var t1: platform.c.timespec = undefined;
+ _ = platform.c.clock_gettime(platform.c.CLOCK.MONOTONIC, &t0);
+ try fwd.run(self.model, &self.scratch, &.{ .{ .cache = &cache, .rows = first }, .{ .cache = &other, .rows = rows - first } }, ids[0..rows], true, .all);
+ _ = platform.c.clock_gettime(platform.c.CLOCK.MONOTONIC, &t1);
+ const ms = @as(f64, @floatFromInt(t1.sec - t0.sec)) * 1e3 + @as(f64, @floatFromInt(t1.nsec - t0.nsec)) / 1e6;
+ cache.len = base;
+ other.len = base;
+ if (trial > 0) best = @min(best, ms);
+ }
+ if (rows == st.window_rows) {
+ shared_base = best;
+ } else self.shared[rows - st.window_rows - 1] = .{ .width = @intCast(rows), .ms = self.costs[st.window_rows - 1].ms + best - shared_base };
+ }
+ self.shared_timed = self.shared.len;
+ std.log.info("LFM2 shared calibration: window16 {d:.2} ms, shared16 {d:.2} ms", .{ self.costs[st.window_rows - 1].ms, shared_base });
+ std.log.info("LFM2 shared marginal prices by rows: 17 {d:.1}, 24 {d:.1}, 32 {d:.1}", .{ self.shared[0].ms, self.shared[7].ms, self.shared[15].ms });
return self.costs.len;
}
pub fn backend(self: *Metal) be.Backend {
- return .{ .ptr = self, .vtable = &.{ .prefill = prefillFn, .first = firstFn, .first_masked = firstMaskedFn, .queue = queueFn, .read = readFn, .verify = verifyFn, .keep = keepFn, .draft = draftFn, .release = releaseFn } };
+ return .{ .ptr = self, .vtable = &.{ .prefill = prefillFn, .prepare_features = prepareFn, .features = featuresFn, .prefill_features = prefillFeaturesFn, .first = firstFn, .first_masked = firstMaskedFn, .queue = queueFn, .read = readFn, .verify = verifyFn, .keep = keepFn, .draft = draftFn, .release = releaseFn } };
}
fn cast(ptr: *anyopaque) *Metal {
@@ -107,8 +150,65 @@ pub const Metal = struct {
return cache;
}
+ /// One forward on the scratch; a stream's tapped rows from earlier forwards are stale after it.
+ fn run(self: *Metal, segments: []const fwd.Segment, tokens: []const u32, record: bool, head: fwd.Head) !void {
+ self.gen += 1;
+ try fwd.run(self.model, &self.scratch, segments, tokens, record, head);
+ }
+
fn prefillFn(ptr: *anyopaque, stream: *lanes.Stream) !void {
+ return cast(ptr).prefillWith(stream, null);
+ }
+
+ /// The prompt pass, `sink` taking each chunk's tapped rows before the chunk's end is kept for reuse.
+ fn prefillFeaturesFn(ptr: *anyopaque, stream: *lanes.Stream, layers: []const u32, sink: be.FeatureSink) !void {
+ const self = cast(ptr);
+ const t = self.scratch.taps orelse return error.UnpreparedLfmTaps;
+ if (layers.len != t.count) return error.UnpreparedLfmTaps;
+ return self.prefillWith(stream, sink);
+ }
+
+ /// A tapped row's bytes: every tap's hidden row, bf16.
+ fn tapBytes(self: *const Metal) usize {
+ return if (self.scratch.taps) |t| t.count * c.hidden * 2 else 0;
+ }
+
+ /// The drafter's layers: every forward from now on keeps their outputs for each row.
+ fn prepareFn(ptr: *anyopaque, layers: []const u32) !void {
const self = cast(ptr);
+ if (layers.len == 0 or layers.len > st.max_taps) return error.BadLfmTaps;
+ for (layers) |l| if (l >= c.layers) return error.BadLfmTaps;
+ if (self.scratch.taps) |t| return if (std.mem.eql(u32, t.layers[0..t.count], layers)) {} else error.BadLfmTaps;
+ var t: st.Taps = .{ .count = @intCast(layers.len), .buf = undefined };
+ @memcpy(t.layers[0..layers.len], layers);
+ t.buf = try self.scratch.memory.alloc(self.scratch.rows * layers.len * c.hidden * 2);
+ self.scratch.taps = t;
+ }
+
+ /// Cache rows [start, start + count) of the stream's latest tapped rows, on the device.
+ fn featuresFn(ptr: *anyopaque, stream: *lanes.Stream, layers: []const u32, start: u64, count: u32) !be.Features {
+ const self = cast(ptr);
+ const t = self.scratch.taps orelse return error.UnpreparedLfmTaps;
+ if (layers.len != t.count) return error.UnpreparedLfmTaps;
+ const cache = try self.cacheOf(stream);
+ const held = cache.tap orelse return error.LfmTapsNotHeld;
+ if (count == 0 or start < held.start or start + count > held.start + held.rows) return error.LfmTapsNotHeld;
+ const width = self.tapBytes();
+ const buf = if (held.gen == 0) cache.last_tap.? else if (held.gen == self.gen) t.buf else return error.StaleLfmTaps;
+ const row = held.row + @as(usize, @intCast(start - held.start));
+ return .{ .buffer = @intFromPtr(buf.id), .offset = row * width, .rows = count, .row_bytes = @intCast(width), .space = .device, .dtype = .bf16 };
+ }
+
+ /// The prompt's last row's taps, copied beside the stream's caches for its first draft (gen 0: kept there).
+ fn keepLastTap(self: *Metal, cache: *st.Cache, row: usize, position: usize) !void {
+ const width = self.tapBytes();
+ if (width == 0) return;
+ if (cache.last_tap == null) cache.last_tap = try cache.memory.alloc(width);
+ @memcpy(cache.last_tap.?.contents()[0..width], (self.scratch.taps.?.buf.contents() + row * width)[0..width]);
+ cache.tap = .{ .start = position, .row = 0, .rows = 1, .gen = 0 };
+ }
+
+ fn prefillWith(self: *Metal, stream: *lanes.Stream, sink: ?be.FeatureSink) !void {
const ids = stream.prompt();
if (ids.len == 0 or ids.len + stream.max_new > self.options.capacity or self.caches.contains(stream)) return error.InvalidLfmPrompt;
const capacity = @min(self.options.capacity, ids.len + stream.max_new + st.window_rows);
@@ -121,7 +221,9 @@ pub const Metal = struct {
stream.cached = 0;
if (stream.reuse.saved) |saved| { // a kept state of this prompt's prefix: the pass starts there
const snap: *snapshot.Snap = @ptrCast(@alignCast(saved));
- if (snap.at < ids.len) if (snapshot.restore(cache, snap)) |_| {
+ // a drafting stream resumes only a state kept with its drafter's context
+ const drafter_less = sink != null and snap.draft == null;
+ if (snap.at < ids.len and !drafter_less) if (snapshot.restore(cache, snap)) |_| {
at = snap.at;
} else |_| {};
stream.cached = @intCast(at);
@@ -136,7 +238,13 @@ pub const Metal = struct {
};
while (mark < stream.reuse.marks.len and stream.reuse.marks[mark] <= at) mark += 1;
if (mark < stream.reuse.marks.len) end = @min(end, stream.reuse.marks[mark]);
- try fwd.run(self.model, &self.scratch, &.{.{ .cache = cache, .rows = end - at }}, ids[at..end], false, .last);
+ try self.run(&.{.{ .cache = cache, .rows = end - at }}, ids[at..end], false, .last);
+ if (sink) |k| {
+ const width = self.tapBytes();
+ const f: be.Features = .{ .buffer = @intFromPtr(self.scratch.taps.?.buf.id), .rows = @intCast(end - at), .row_bytes = @intCast(width), .space = .device, .dtype = .bf16 };
+ try k.take(k.ptr, stream, f, at);
+ if (end == ids.len) try self.keepLastTap(cache, end - 1 - at, end - 1);
+ }
at = end;
if (stream.reuse.hook) |hk| if (std.mem.indexOfScalar(u32, stream.reuse.marks, @intCast(at)) != null)
hk.at(hk.ptr, stream, @intCast(at));
@@ -165,7 +273,8 @@ pub const Metal = struct {
.value => |v| v,
.handle => |h| @intCast(h),
};
- try fwd.run(self.model, &self.scratch, &.{.{ .cache = cache, .rows = 1 }}, &.{token}, false, .all);
+ try self.run(&.{.{ .cache = cache, .rows = 1 }}, &.{token}, false, .all);
+ if (self.scratch.taps != null) cache.tap = .{ .start = cache.len - 1, .row = 0, .rows = 1, .gen = self.gen };
return try draw(self.gpa, cache.logits.slice(u16, c.vocab), stream.sampling, position, &.{});
}
@@ -178,25 +287,43 @@ pub const Metal = struct {
const self = cast(ptr);
var segments: [st.batch_rows]fwd.Segment = undefined;
var ids: [st.batch_rows]u32 = undefined;
+ var held: [st.batch_rows]fwd.Held = undefined;
+ var count: usize = 0;
+ var wait: ?fwd.Wait = null;
var total: usize = 0;
if (windows.len > segments.len or windows.len != out.len) return error.InvalidLfmWindows;
for (windows, 0..) |w, i| {
const rows = w.rows();
- if (w.parents != null or w.held != 0 or rows > st.window_rows or total + rows > st.batch_rows or w.positions.len != rows) return error.UnsupportedLfmWindow;
+ if (w.held != 0) { // the drafter's picks, copied into the window's rows on the GPU
+ const source = self.held_source orelse return error.UnsupportedLfmWindow;
+ if (total + rows > st.batch_rows) return error.UnsupportedLfmWindow;
+ if (source.get(source.ptr, w.stream)) |dev| {
+ held[count] = .{ .row = total + 1, .src = dev.src, .off = dev.off, .stride = dev.stride, .count = w.held };
+ count += 1;
+ wait = if (wait) |x| (if (x.value >= dev.wait.value) x else dev.wait) else dev.wait;
+ @memset(ids[total + 1 ..][0..w.held], 0);
+ } else try source.host(source.ptr, w.stream, ids[total + 1 ..][0..w.held]);
+ }
+ if (w.parents != null or rows > st.window_rows or total + rows > st.batch_rows or w.positions.len != rows) return error.UnsupportedLfmWindow;
if (w.masks.len != 0 and w.masks.len != rows * mask_words) return error.InvalidLfmMasks;
const cache = try self.cacheOf(w.stream);
for (w.positions, 0..) |p, r| if (p != cache.len + r + 1) return error.InvalidLfmPosition;
segments[i] = .{ .cache = cache, .rows = rows };
ids[total] = w.pending;
- @memcpy(ids[total + 1 ..][0..w.tokens.len], w.tokens);
+ @memcpy(ids[total + 1 + w.held ..][0..w.tokens.len], w.tokens);
total += rows;
}
- try fwd.run(self.model, &self.scratch, segments[0..windows.len], ids[0..total], true, .all);
+ self.gen += 1;
+ try fwd.runHeld(self.model, &self.scratch, segments[0..windows.len], ids[0..total], true, .all, held[0..count], wait);
+ if (self.scratch.taps != null) for (segments[0..windows.len]) |seg| {
+ const last = seg.cache.last.?;
+ seg.cache.tap = .{ .start = last.start, .row = last.base, .rows = last.rows, .gen = self.gen };
+ };
const logits = self.scratch.logits.slice(u16, total * c.vocab);
const best = self.scratch.best.slice(u32, 2 * total);
var base: usize = 0;
for (windows, out) |w, o| {
- if (o.sampled.len != w.rows() or o.drafts.len != w.tokens.len) return error.InvalidLfmOutputs;
+ if (o.sampled.len != w.rows() or o.drafts.len != w.rows() - 1) return error.InvalidLfmOutputs;
for (o.sampled, w.positions, 0..) |*t, p, r| {
const mask = if (w.masks.len > 0) w.masks[r * mask_words ..][0..mask_words] else &.{};
// A greedy row takes the GPU's pick, the host's draw on the same logits; a nonfinite row errors there.
@@ -207,7 +334,7 @@ pub const Metal = struct {
}
t.* = try draw(self.gpa, logits[(base + r) * c.vocab ..][0..c.vocab], w.stream.sampling, p, mask);
}
- @memcpy(o.drafts, w.tokens);
+ @memcpy(o.drafts, self.scratch.ids.slice(u32, base + w.rows())[base + 1 ..][0 .. w.rows() - 1]);
base += w.rows();
}
}
@@ -381,3 +508,15 @@ test "a masked draw is the full-vocabulary draw with the other tokens at -inf" {
@memset(&mask, 0);
try std.testing.expectError(error.NoAllowedLfmToken, draw(gpa, words, null, 0, &mask));
}
+
+test "measured shared rows price the batch without widening a stream's qualified window" {
+ var backend_: Metal = .{ .gpa = std.testing.allocator, .model = undefined, .options = .{ .capacity = 512 }, .scratch = undefined, .width = st.window_rows, .timed = 1, .shared_timed = 1 };
+ backend_.costs[0] = .{ .width = 1, .ms = 2 };
+ backend_.shared[0] = .{ .width = 17, .ms = 9 };
+ var cfg = try lanes.Config.init(std.testing.allocator, backend_.facts(), st.window_rows, st.window_rows - 1);
+ defer cfg.deinit(std.testing.allocator);
+ try std.testing.expectEqual(@as(u32, st.window_rows), cfg.family_width);
+ try std.testing.expectEqual(@as(?f64, 2), cfg.family_costs.get(1));
+ try std.testing.expectEqual(@as(?f64, null), cfg.family_costs.get(17));
+ try std.testing.expectEqual(@as(?f64, 9), cfg.shared_costs.get(17));
+}
diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig
new file mode 100644
index 0000000000..df55a38e9a
--- /dev/null
+++ b/zig/src/families/lfm2_moe/dspark.zig
@@ -0,0 +1,578 @@
+//! LiquidAI's DSpark drafter on LFM2's taps: a stream's context keys, a block of rows, a Markov chain of drafts.
+const std = @import("std");
+const mtl = @import("metal");
+const lanes = @import("lanes");
+const c = @import("config.zig");
+const dc = @import("dspark_config.zig");
+const dw = @import("dspark_weights.zig");
+const block = @import("dspark_block.zig");
+const st = @import("state.zig");
+const snapshot = @import("snapshot.zig");
+const Model = @import("model.zig").Model;
+const backend = @import("backend.zig");
+
+const dr = lanes.drafter;
+const Stream = lanes.Stream;
+const shared = mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked;
+
+/// Block rows one hold pass takes (the decode twin's rows), and context rows one absorb pass takes.
+pub const pass_rows = 32;
+pub const absorb_rows = 2048;
+/// Verified rows a lane keeps on the host until its next hold absorbs them (a round's rows for no GPU work).
+pub const stage_rows = 128;
+/// The block counts timed at load, at every depth that fits one pass and the checkpoint's block.
+const hold_grid = [_]u32{ 1, 2, 4, 8, 16 };
+const kw = c.kv_heads * c.head_dim;
+
+/// `bits`: 0 runs the bf16 matrices as stored, 8 quantizes them at load.
+pub const Options = struct { capacity: usize, bits: u8 = 0 };
+
+/// One stream's context keys and values a layer ([kv head][cap][head dim], as the target's attention reads them).
+pub const Lane = struct {
+ mem: mtl.Buffer, // each layer's keys then values, then the staged rows (shared storage)
+ cap: usize, // positions each holds: the context and one block past it
+ stage_at: usize, // the staged rows' byte offset in `mem`
+ end: u64 = 0, // context rows absorbed: positions [0, end)
+ staged: u32 = 0, // tapped rows [end, end + staged), not absorbed yet
+ held: [16]u32 = @splat(0),
+ held_len: u32 = 0,
+ conf: [16]f32 = @splat(0), // each held draft's chance given the drafts before it landed (the confidence head)
+ conf_len: u32 = 0, // the drafts `conf` covers: the last settled hold's depth
+ slot: ?u32 = null, // its block's place in the last signalled hold, whose picks still hold its drafts
+
+ /// Layer `layer`'s keys, or its values when `values`, in `mem`.
+ pub fn kv(l: *const Lane, layer: usize, values: bool) block.Ref {
+ return .{ .buf = l.mem, .off = (2 * layer + @intFromBool(values)) * l.cap * kw * 2 };
+ }
+};
+
+/// The passes' working buffers, one pass at a time on the target's queue.
+pub const Scratch = struct {
+ ids: mtl.Buffer,
+ h: mtl.Buffer,
+ x: mtl.Buffer,
+ r: mtl.Buffer,
+ y: mtl.Buffer,
+ q: mtl.Buffer,
+ k: mtl.Buffer,
+ v: mtl.Buffer,
+ queries: mtl.Buffer,
+ pm: mtl.Buffer,
+ pl: mtl.Buffer,
+ po: mtl.Buffer,
+ gate: mtl.Buffer,
+ up: mtl.Buffer,
+ act: mtl.Buffer,
+ logits: mtl.Buffer,
+ sums: mtl.Buffer,
+ fcx: mtl.Buffer, // an absorb pass's context rows: fc's output, normed in place
+ dump: mtl.Buffer, // the query heads lfm2_qk rotates from context rows, which no one reads
+ best: mtl.Buffer, // the Markov scan's best score and token of each threadgroup, by stream
+ at: mtl.Buffer,
+ picks: mtl.Buffer,
+ conf: mtl.Buffer, // the confidence logits, step by step, a pass's blocks each
+};
+
+pub const Drafter = struct {
+ gpa: std.mem.Allocator,
+ io: std.Io,
+ target: *Model,
+ config: dc.Config,
+ w: dw.Weights,
+ options: Options,
+ s: Scratch,
+ owned: std.ArrayList(mtl.Buffer) = .empty,
+ lanes: std.AutoHashMapUnmanaged(*Stream, *Lane) = .empty,
+ /// One stream's hold wall ms at full depth, and that over the block (the depth rule's cost a draft).
+ hold_ms: f64 = 0,
+ step_ms: f64 = 0,
+ holds: [2 * pass_rows]lanes.hold_price.HoldCost = undefined, // the power-of-two block counts sum to fewer than 2 * pass_rows depths
+ holds_len: usize = 0,
+ event: mtl.Event = undefined,
+ signalled: u64 = 0,
+ pending: ?mtl.CommandBuffer = null,
+ pass: [pass_rows]*Lane = undefined, // the signalled hold's lanes and depths, in its blocks' order
+ depths: [pass_rows]u32 = undefined,
+ pass_len: usize = 0,
+
+ pub fn init(gpa: std.mem.Allocator, io: std.Io, target: *Model, dir: []const u8, options: Options) !*Drafter {
+ const d = try gpa.create(Drafter);
+ errdefer gpa.destroy(d);
+ d.* = .{ .gpa = gpa, .io = io, .target = target, .config = try dc.Config.read(gpa, io, dir), .w = undefined, .options = options, .s = undefined };
+ d.event = try target.device.event();
+ errdefer d.event.deinit();
+ d.w = try dw.load(gpa, target.device, io, dir, &d.config, options.bits);
+ errdefer d.w.deinit(gpa);
+ try d.prepare();
+ return d;
+ }
+
+ fn prepare(d: *Drafter) !void {
+ errdefer {
+ for (d.owned.items) |b| b.deinit();
+ d.owned.deinit(d.gpa);
+ }
+ try d.scratch();
+ try d.warm();
+ }
+
+ pub fn deinit(d: *Drafter) void {
+ d.settle() catch {};
+ d.event.deinit();
+ var it = d.lanes.valueIterator();
+ while (it.next()) |l| d.dropLane(l.*);
+ d.lanes.deinit(d.gpa);
+ for (d.owned.items) |b| b.deinit();
+ d.owned.deinit(d.gpa);
+ d.w.deinit(d.gpa);
+ d.gpa.destroy(d);
+ }
+
+ fn buffer(d: *Drafter, bytes: usize) !mtl.Buffer {
+ const b = try d.target.device.buffer(@max(bytes, 16), shared);
+ errdefer b.deinit();
+ try d.owned.append(d.gpa, b);
+ return b;
+ }
+
+ fn scratch(d: *Drafter) !void {
+ const r = pass_rows;
+ const wide = @max(pass_rows, absorb_rows);
+ const i = d.config.intermediate;
+ const partials = d.partRows() * st.subs * c.query_heads * st.chunks;
+ const sums = @max(absorb_rows * (d.config.tap_count * c.hidden / 64), r * (i / 64));
+ d.s = .{
+ .ids = try d.buffer(r * 4),
+ .h = try d.buffer(r * c.hidden * 2),
+ .x = try d.buffer(r * c.hidden * 2),
+ .r = try d.buffer(r * c.hidden * 2),
+ .y = try d.buffer(r * c.hidden * 2),
+ .q = try d.buffer(r * c.hidden * 2),
+ .k = try d.buffer(wide * kw * 2),
+ .v = try d.buffer(wide * kw * 2),
+ .queries = try d.buffer((r + st.attn_rows) * c.hidden * 2),
+ .pm = try d.buffer(partials * 4),
+ .pl = try d.buffer(partials * 4),
+ .po = try d.buffer(partials * c.head_dim * 4),
+ .gate = try d.buffer(r * i * 2),
+ .up = try d.buffer(r * i * 2),
+ .act = try d.buffer(r * i * 2),
+ .logits = try d.buffer(r * c.vocab * 2),
+ .sums = try d.buffer(sums * 4),
+ .fcx = try d.buffer(absorb_rows * c.hidden * 2),
+ .dump = try d.buffer(absorb_rows * c.hidden * 2),
+ .best = try d.buffer(r * block.scan_groups * 4),
+ .at = try d.buffer(r * block.scan_groups * 4),
+ .picks = try d.buffer(d.config.block * r * 8),
+ .conf = try d.buffer(d.config.block * r * 4),
+ };
+ }
+
+ /// Query rows the attention partials hold at once: a whole hold pass's, so its blocks attend side by side.
+ pub fn partRows(d: *const Drafter) usize {
+ return @max(pass_rows, d.config.block);
+ }
+
+ /// The drafter's buffers that every round reads (weights and scratch), for the server's residency set.
+ pub fn buffers(d: *const Drafter, gpa: std.mem.Allocator) ![]mtl.Buffer {
+ return std.mem.concat(gpa, mtl.Buffer, &.{ d.w.buffers(), d.owned.items });
+ }
+
+ pub fn drafter(d: *Drafter) dr.Drafter {
+ return .{ .ptr = d, .vtable = &.{ .taps = tapsFn, .facts = factsFn, .open = openFn, .absorb = absorbFn, .hold = holdFn, .held = heldFn, .release = releaseFn, .chances = chancesFn } };
+ }
+
+ /// Whether holds compute the confidence head: the verify rows follow its chances, else the core's depth rule.
+ pub fn confident(d: *const Drafter) bool {
+ return d.w.confidence != null;
+ }
+
+ fn cast(ptr: *anyopaque) *Drafter {
+ return @ptrCast(@alignCast(ptr));
+ }
+
+ fn tapsFn(ptr: *anyopaque) []const u32 {
+ const d = cast(ptr);
+ return d.config.taps[0..d.config.tap_count];
+ }
+
+ /// Acceptance a depth until a stream has its own: the card's 5.16 tokens a step at block 9 is 0.84 a depth.
+ const prior: [16]f64 = @splat(0.84);
+
+ fn factsFn(ptr: *anyopaque) dr.Facts {
+ const d = cast(ptr);
+ var f: dr.Facts = .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .stream_bytes = d.laneBytes(d.options.capacity), .copy_first = true };
+ f.chances = d.confident();
+ f.hold_costs = d.holds[0..d.holds_len];
+ return f;
+ }
+
+ /// A held chain's chances: the confidence head's running product, or for a fixed window, ones that fall by depth.
+ fn chancesFn(ptr: *anyopaque, s: *Stream, out: []f64) anyerror!bool {
+ const d = cast(ptr);
+ const l = d.lanes.get(s) orelse return false;
+ if (!d.confident()) return false;
+ if (d.pending != null and l.slot != null and l.conf_len > 0) {
+ // the hold still runs: the lane's last settled confidences price it, so the host does not wait on the GPU
+ if (out.len > d.depths[l.slot.?]) return error.DraftHeldFewer;
+ var reach: f64 = 1;
+ for (out, 0..) |*o, j| {
+ reach *= l.conf[@min(j, l.conf_len - 1)];
+ o.* = reach;
+ }
+ return true;
+ }
+ try d.settle();
+ if (out.len > l.held_len) return error.DraftHeldFewer;
+ var reach: f64 = 1;
+ for (out, l.conf[0..out.len]) |*o, p| {
+ reach *= p;
+ o.* = reach;
+ }
+ return true;
+ }
+
+ /// A stream's lane at `capacity` positions: its context keys and values a layer, and its staged rows.
+ pub fn laneBytes(d: *const Drafter, capacity: usize) usize {
+ return @intCast(d.keptBytes(capacity + d.config.block) + stage_rows * d.tapBytes());
+ }
+
+ /// A tapped row's bytes: every tap's hidden row, bf16.
+ pub fn tapBytes(d: *const Drafter) usize {
+ return d.config.tap_count * c.hidden * 2;
+ }
+
+ /// The lane's staged rows as an absorb piece (null: none staged).
+ pub fn stagedPiece(l: *Lane) ?block.Piece {
+ if (l.staged == 0) return null;
+ return .{ .lane = l, .src = .{ .buf = l.mem, .off = l.stage_at }, .rows = l.staged, .p0 = l.end };
+ }
+
+ /// The lane's staged rows absorbed now, in a command buffer of their own.
+ fn flush(d: *Drafter, l: *Lane) !void {
+ const pc = stagedPiece(l) orelse return;
+ const pool = mtl.objc.Pool.push();
+ defer pool.pop();
+ const run = block.begin(d);
+ try block.absorbPass(d, run.e, &.{pc});
+ try block.end(run.cb, run.e);
+ l.end += l.staged;
+ l.staged = 0;
+ }
+
+ fn openFn(ptr: *anyopaque, s: *Stream) anyerror!void {
+ const d = cast(ptr);
+ if (d.lanes.contains(s)) return error.DraftStreamOpen;
+ const want = s.prompt_len + s.max_new + 2 * d.config.block;
+ const l = try d.newLane(@min(d.options.capacity, (want + 63) / 64 * 64));
+ errdefer d.dropLane(l);
+ try d.lanes.put(d.gpa, s, l);
+ }
+
+ /// A lane in one buffer of laneBytes(capacity).
+ pub fn newLane(d: *Drafter, capacity: usize) !*Lane {
+ const l = try d.gpa.create(Lane);
+ errdefer d.gpa.destroy(l);
+ const at: usize = @intCast(d.keptBytes(capacity + d.config.block));
+ l.* = .{ .mem = try d.target.device.buffer(d.laneBytes(capacity), shared), .cap = capacity + d.config.block, .stage_at = at };
+ return l;
+ }
+
+ pub fn dropLane(d: *Drafter, l: *Lane) void {
+ l.mem.deinit();
+ d.gpa.destroy(l);
+ }
+
+ fn releaseFn(ptr: *anyopaque, s: *Stream) void {
+ const d = cast(ptr);
+ d.unpass() catch {};
+ if (d.lanes.fetchRemove(s)) |e| d.dropLane(e.value);
+ }
+
+ fn heldFn(ptr: *anyopaque, streams: []const *Stream, out: []const []u32) anyerror!void {
+ const d = cast(ptr);
+ try d.settle();
+ for (streams, out) |s, o| {
+ const l = d.lanes.get(s) orelse return error.DraftStreamUnknown;
+ if (o.len > l.held_len) return error.DraftHeldFewer;
+ @memcpy(o, l.held[0..o.len]);
+ }
+ }
+
+ /// Bytes a lane's context at `at` takes beside a kept prompt state.
+ pub fn keptBytes(d: *const Drafter, at: usize) u64 {
+ return 2 * @as(u64, d.config.layers) * at * kw * 2;
+ }
+
+ /// Stream `s`'s context copied into `snap` when its lane stands at the snapshot's position (a prompt chunk's end).
+ pub fn keepLane(d: *Drafter, s: *Stream, snap: *snapshot.Snap) !void {
+ const l = d.lanes.get(s) orelse return;
+ try d.settle();
+ try d.flush(l);
+ if (l.end != snap.at) return;
+ const bytes = try d.gpa.alloc(u8, @intCast(d.keptBytes(snap.at)));
+ copyLane(d, l, snap.at, bytes, true);
+ snap.draft = bytes;
+ }
+
+ /// The lane takes the context kept with the state its stream's prompt pass resumed at `start`.
+ fn resumeLane(d: *Drafter, s: *Stream, l: *Lane, start: u64) !void {
+ const saved = s.reuse.saved orelse return error.DraftContextGap;
+ const snap: *const snapshot.Snap = @ptrCast(@alignCast(saved));
+ const bytes = snap.draft orelse return error.DraftContextGap;
+ if (s.cached != start or snap.at != start or l.end != 0 or start + d.config.block > l.cap) return error.DraftContextGap;
+ copyLane(d, l, snap.at, bytes, false);
+ l.end = start;
+ }
+
+ /// Rows [0, at) of every layer's keys and values and `bytes` (`out`: the lane's into bytes), kv head by kv head.
+ fn copyLane(d: *const Drafter, l: *const Lane, at: usize, bytes: []u8, out: bool) void {
+ const n = at * c.head_dim * 2;
+ var off: usize = 0;
+ for (0..d.config.layers) |i| for ([_]block.Ref{ l.kv(i, false), l.kv(i, true) }) |r| for (0..c.kv_heads) |h| {
+ const live = (r.buf.contents() + r.off + h * l.cap * c.head_dim * 2)[0..n];
+ if (out) @memcpy(bytes[off..][0..n], live) else @memcpy(live, bytes[off..][0..n]);
+ off += n;
+ };
+ }
+
+ /// Rows from `start` replace the context there: rows that fit the stage wait on the host for the next hold.
+ fn absorbFn(ptr: *anyopaque, items: []const dr.Absorb) anyerror!void {
+ const d = cast(ptr);
+ try d.settle();
+ const width = d.tapBytes();
+ const pool = mtl.objc.Pool.push();
+ defer pool.pop();
+ var run: ?@TypeOf(block.begin(d)) = null;
+ var pieces: [32]block.Piece = undefined;
+ var count: usize = 0;
+ var rows: usize = 0;
+ for (items) |it| {
+ const l = d.lanes.get(it.stream) orelse return error.DraftStreamUnknown;
+ const f = it.features;
+ if (f.space != .device or f.dtype != .bf16 or f.row_bytes != width or it.rows != null) return error.DraftFeaturesUnsupported;
+ if (it.start > l.end + l.staged) try d.resumeLane(it.stream, l, it.start); // a prompt pass that resumed
+ if (it.start + f.rows + d.config.block > l.cap) return error.DraftContextGap;
+ l.held_len = 0;
+ l.slot = null;
+ const src = mtl.Buffer{ .id = @ptrFromInt(f.buffer) };
+ // rows from `start` replace what was staged past it; rows before the absorbed end go to the GPU now
+ const keep: u32 = if (it.start >= l.end) @intCast(it.start - l.end) else 0;
+ l.staged = @min(l.staged, keep);
+ if (it.start >= l.end and l.staged + f.rows <= stage_rows) {
+ const from = (src.contents() + f.offset)[0 .. f.rows * width];
+ @memcpy((l.mem.contents() + l.stage_at + l.staged * width)[0 .. f.rows * width], from);
+ l.staged += f.rows;
+ continue;
+ }
+ if (run == null) run = block.begin(d);
+ const e = run.?.e;
+ if (stagedPiece(l)) |pc| {
+ if (count == pieces.len or rows + pc.rows > absorb_rows) {
+ try block.absorbPass(d, e, pieces[0..count]);
+ count = 0;
+ rows = 0;
+ }
+ pieces[count] = pc;
+ count += 1;
+ rows += pc.rows;
+ }
+ var at: usize = 0;
+ while (at < f.rows) {
+ if (rows == absorb_rows or count == pieces.len) { // the pass is full: encode it
+ try block.absorbPass(d, e, pieces[0..count]);
+ count = 0;
+ rows = 0;
+ }
+ const n = @min(absorb_rows - rows, f.rows - at);
+ pieces[count] = .{ .lane = l, .src = .{ .buf = src, .off = f.offset + at * width }, .rows = n, .p0 = it.start + at };
+ count += 1;
+ rows += n;
+ at += n;
+ }
+ l.end = it.start + f.rows;
+ l.staged = 0;
+ }
+ const r = run orelse return;
+ // the other lanes' staged rows join the pass: one waited pass a stage's worth of rounds, not one a stream
+ var others = d.lanes.valueIterator();
+ while (others.next()) |lp| {
+ const pc = stagedPiece(lp.*) orelse continue;
+ if (count == pieces.len or rows + pc.rows > absorb_rows) {
+ try block.absorbPass(d, r.e, pieces[0..count]);
+ count = 0;
+ rows = 0;
+ }
+ pieces[count] = pc;
+ count += 1;
+ rows += pc.rows;
+ lp.*.end += lp.*.staged;
+ lp.*.staged = 0;
+ }
+ if (count > 0) try block.absorbPass(d, r.e, pieces[0..count]);
+ try block.end(r.cb, r.e);
+ }
+
+ /// Each stream's chain of `depth` drafts after `pending` at `position`, its block's rows in a pass of pass_rows.
+ fn holdFn(ptr: *anyopaque, items: []const dr.Hold) anyerror!void {
+ const d = cast(ptr);
+ try d.unpass();
+ var at: usize = 0;
+ while (at < items.len) {
+ // as many streams as their blocks' rows fit a pass: each block is as long as the pass's deepest chain
+ var deepest: usize = 0;
+ var end = at;
+ while (end < items.len) : (end += 1) {
+ const rows = @max(deepest, items[end].depth);
+ if ((end - at + 1) * rows > pass_rows) break;
+ deepest = rows;
+ }
+ if (end == at) return error.DraftDepth;
+ var blocks: [pass_rows]block.Block = undefined;
+ const group = items[at..end];
+ at = end;
+ for (group, blocks[0..group.len]) |it, *b| {
+ const l = d.lanes.get(it.stream) orelse return error.DraftStreamUnknown;
+ if (it.depth == 0 or it.depth > d.config.block or it.position == 0) return error.DraftDepth;
+ const anchor = it.position - 1;
+ if (l.end + l.staged != anchor or anchor + d.config.block > l.cap) return error.DraftContextGap;
+ b.* = .{ .lane = l, .pending = it.pending, .anchor = anchor, .depth = it.depth };
+ }
+ if (group.len == items.len) { // one pass: signalled, not waited on
+ d.signalled += 1;
+ d.pending = try block.hold(d, blocks[0..group.len], .{ .event = d.event, .value = d.signalled });
+ for (blocks[0..group.len], 0..) |b, bi| {
+ d.pass[bi] = b.lane;
+ d.depths[bi] = b.depth;
+ b.lane.slot = @intCast(bi);
+ b.lane.held_len = 0;
+ }
+ d.pass_len = group.len;
+ continue;
+ }
+ _ = try block.hold(d, blocks[0..group.len], null);
+ }
+ }
+
+ /// The signalled hold waited on, its picks read into its lanes' held chains (nothing pending: nothing done).
+ fn settle(d: *Drafter) !void {
+ const cb = d.pending orelse return;
+ d.pending = null;
+ defer cb.release();
+ cb.wait();
+ if (cb.failure()) |msg| {
+ std.log.err("lfm2 dspark: a hold failed on the GPU: {s}", .{msg});
+ return error.DraftGpuFailed;
+ }
+ d.priceHold((cb.gpuEnd() - cb.gpuStart()) * 1000);
+ try block.readPicks(d, d.pass[0..d.pass_len], d.depths[0..d.pass_len]);
+ }
+
+ /// A settled hold's GPU ms against its price: the timed passes at its block counts move a tenth of the way to it.
+ pub fn priceHold(d: *Drafter, ms: f64) void {
+ const timed = d.holds[0..d.holds_len];
+ if (timed.len == 0 or d.pass_len == 0 or !(ms > 0)) return;
+ const deepest = std.mem.max(u32, d.depths[0..d.pass_len]);
+ const priced = lanes.hold_price.holdMs(timed, d.pass_len, deepest);
+ if (!(priced > 0)) return;
+ const step = 1 + 0.1 * (ms / priced - 1);
+ var lo: u32 = 0;
+ var hi: u32 = std.math.maxInt(u32);
+ for (timed) |h| {
+ if (h.streams <= d.pass_len) lo = @max(lo, h.streams);
+ if (h.streams >= d.pass_len) hi = @min(hi, h.streams);
+ }
+ for (timed) |*h| if (h.streams >= lo and h.streams <= hi) {
+ h.ms *= step;
+ };
+ }
+
+ /// The signalled hold settled and its lanes' picks forgotten (a lane may be dropped, or a hold may overwrite).
+ fn unpass(d: *Drafter) !void {
+ defer {
+ for (d.pass[0..d.pass_len]) |l| l.slot = null;
+ d.pass_len = 0;
+ }
+ try d.settle();
+ }
+
+ /// Where stream `s`'s drafts from the last signalled hold lie in the picks, for a verify to copy on the GPU.
+ pub fn device(ptr: *anyopaque, s: *Stream) ?backend.Device {
+ const d = cast(ptr);
+ const l = d.lanes.get(s) orelse return null;
+ const slot = l.slot orelse return null;
+ return .{ .src = d.s.picks, .off = slot * 8, .stride = pass_rows * 2, .wait = .{ .event = d.event, .value = d.signalled } };
+ }
+
+ /// The target's verify reading the held drafts on the device, or as host tokens where it cannot.
+ pub fn heldSource(d: *Drafter) backend.HeldSource {
+ return .{ .ptr = d, .get = device, .host = hostHeld };
+ }
+
+ fn hostHeld(ptr: *anyopaque, s: *Stream, out: []u32) anyerror!void {
+ return heldFn(ptr, &.{s}, &.{out});
+ }
+
+ fn mibOf(bufs: []const mtl.Buffer) f64 {
+ var n: usize = 0;
+ for (bufs) |b| n += b.length();
+ return @as(f64, @floatFromInt(n)) / (1024.0 * 1024.0);
+ }
+
+ /// Holds on empty contexts: the pipelines compiled, then holds timed by blocks and depth (best of three).
+ fn warm(d: *Drafter) !void {
+ var made: [hold_grid[hold_grid.len - 1]]*Lane = undefined;
+ var n: usize = 0;
+ defer for (made[0..n]) |l| d.dropLane(l);
+ while (n < made.len) : (n += 1) made[n] = try d.newLane(64);
+ const l = made[0];
+ var best: [2]f64 = @splat(std.math.floatMax(f64));
+ for (0..4) |i| for ([_]u32{ 1, d.config.block }, &best) |depth, *b| {
+ const t0 = std.Io.Clock.awake.now(d.io);
+ _ = try block.hold(d, &.{.{ .lane = l, .pending = 1, .anchor = 0, .depth = depth }}, null);
+ const ms = @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(d.io)).toNanoseconds())) / 1e6;
+ if (i > 0) b.* = @min(b.*, ms);
+ };
+ d.hold_ms = best[1];
+ d.step_ms = best[1] / @as(f64, @floatFromInt(d.config.block));
+ std.log.info("lfm2 dspark: a 1-draft hold takes {d:.2} ms, a {d}-draft hold {d:.2} ms", .{ best[0], d.config.block, best[1] });
+ std.log.info("lfm2 dspark: weights {d:.1} MiB, scratch {d:.1} MiB, a stream {d} KiB a position and {d} KiB staged", .{ mibOf(d.w.buffers()), mibOf(d.owned.items), d.keptBytes(1) / 1024, stage_rows * d.tapBytes() / 1024 });
+ var blocks: [hold_grid[hold_grid.len - 1]]block.Block = undefined;
+ for (made, &blocks) |lane, *b| b.* = .{ .lane = lane, .pending = 1, .anchor = 0, .depth = 1 };
+ d.holds_len = 0;
+ for (hold_grid) |streams| for (1..@min(d.config.block, pass_rows / streams) + 1) |depth| {
+ for (blocks[0..streams]) |*b| b.depth = @intCast(depth);
+ var ms_best = std.math.floatMax(f64);
+ for (0..4) |i| {
+ const t0 = std.Io.Clock.awake.now(d.io);
+ _ = try block.hold(d, blocks[0..streams], null);
+ const ms = @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(d.io)).toNanoseconds())) / 1e6;
+ if (i > 0) ms_best = @min(ms_best, ms);
+ }
+ d.holds[d.holds_len] = .{ .streams = streams, .depth = @intCast(depth), .ms = ms_best };
+ d.holds_len += 1;
+ };
+ }
+};
+
+test "lfm2 dspark: scratch and warm allocation failures release buffers before their list" {
+ const device = mtl.Device.init() catch return error.SkipZigTest;
+ defer device.deinit();
+ var target: Model = undefined;
+ target.device = device;
+ for ([_]usize{ 1, 3 }) |fail_index| {
+ var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = fail_index });
+ var d: Drafter = .{
+ .gpa = failing.allocator(),
+ .io = undefined,
+ .target = &target,
+ .config = try dc.Config.parse(std.testing.allocator, dc.stock),
+ .w = undefined,
+ .options = .{ .capacity = 64 },
+ .s = undefined,
+ };
+ try std.testing.expectError(error.OutOfMemory, d.prepare());
+ try std.testing.expect(failing.has_induced_failure);
+ try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes);
+ }
+}
diff --git a/zig/src/families/lfm2_moe/dspark_bf16_test.zig b/zig/src/families/lfm2_moe/dspark_bf16_test.zig
new file mode 100644
index 0000000000..293549bde1
--- /dev/null
+++ b/zig/src/families/lfm2_moe/dspark_bf16_test.zig
@@ -0,0 +1,121 @@
+//! The DSpark drafter's bf16 kernels on the GPU against host references: the linears and the Markov scan over bf16 W2.
+const std = @import("std");
+const mtl = @import("metal");
+const quant = @import("dspark_quant.zig");
+const base = @import("dspark_test.zig");
+
+const Args = extern struct { rows: u32, n: u32, k: u32 };
+
+/// y = x W^T through `key` on random bf16 data, each output within a bf16 step of the fp64 product.
+fn linear(comptime key: []const u8, rows: usize, n: usize, k: usize, groups: [2]usize, threads: usize) !void {
+ var r = try base.Rig.init();
+ defer r.deinit();
+ const x = try r.buffer(u16, rows * k);
+ defer x.deinit();
+ const w = try r.buffer(u16, n * k);
+ defer w.deinit();
+ const y = try r.buffer(u16, rows * n);
+ defer y.deinit();
+ base.noise(31, x.slice(u16, rows * k), 2);
+ base.noise(32, w.slice(u16, n * k), 0.5);
+ const cb = r.queue.commandBuffer();
+ const e = cb.compute(.serial);
+ e.setPipeline(r.k.get(key));
+ e.setBuffer(x, 0, 0);
+ e.setBuffer(w, 0, 1);
+ e.setValue(Args{ .rows = @intCast(rows), .n = @intCast(n), .k = @intCast(k) }, 2);
+ e.setBuffer(y, 0, 3);
+ e.dispatchGroups(mtl.Size.of(groups[0], groups[1], 1), mtl.Size.of(threads, 1, 1));
+ e.end();
+ cb.commit();
+ cb.wait();
+ if (cb.failure()) |_| return error.GpuFailed;
+ const xs = x.slice(u16, rows * k);
+ const ws = w.slice(u16, n * k);
+ const ys = y.slice(u16, rows * n);
+ for (0..rows) |i| for (0..n) |j| {
+ var dot: f64 = 0;
+ for (0..k) |t| dot += @as(f64, quant.toF32(xs[i * k + t])) * @as(f64, quant.toF32(ws[j * k + t]));
+ const got: f64 = quant.toF32(ys[i * n + j]);
+ try std.testing.expect(@abs(got - dot) <= @abs(dot) * 0x1p-7 + 1e-3);
+ };
+}
+
+test "lfm2 dspark: the bf16 rows kernel is x W^T for one row, a full 16 and a ragged 19 (two row groups)" {
+ for ([_]usize{ 1, 16, 19 }) |rows| try linear("lfm2_bf16_rows", rows, 100, 288, .{ 13, (rows + 15) / 16 }, 256);
+}
+
+test "lfm2 dspark: the bf16 tile kernel is x W^T for a ragged 70 rows and 100 columns, k a multiple of 32" {
+ try linear("lfm2_bf16_mma", 70, 100, 288, .{ 2, 2 }, 128);
+}
+
+test "lfm2 dspark: the bf16 tile kernel at the drafter's widths" {
+ try linear("lfm2_bf16_mma", 33, 512, 2048, .{ 8, 1 }, 128);
+}
+
+test "lfm2 dspark: a Markov step over bf16 W2 picks the argmax of bf16(base + bf16(W2 . W1[prev])), a NaN row flagged" {
+ var r = try base.Rig.init();
+ defer r.deinit();
+ const rank = 256;
+ const vocab = 1300;
+ const streams = 4;
+ const groups = (vocab + 511) / 512;
+ const w1 = try r.buffer(u16, vocab * rank);
+ defer w1.deinit();
+ base.noise(11, w1.slice(u16, vocab * rank), 1);
+ const w2 = try r.buffer(u16, vocab * rank);
+ defer w2.deinit();
+ base.noise(12, w2.slice(u16, vocab * rank), 0.5);
+ const ids = try r.buffer(u32, 2 * streams);
+ defer ids.deinit();
+ @memcpy(ids.slice(u32, 2 * streams), &[_]u32{ 17, 9, vocab - 1, 9, vocab + 5, 0, 3, 3 });
+ const logits = try r.buffer(u16, streams * 2 * vocab);
+ defer logits.deinit();
+ base.noise(14, logits.slice(u16, streams * 2 * vocab), 4);
+ logits.slice(u16, streams * 2 * vocab)[3 * 2 * vocab + 700] = 0x7fc0;
+ const best = try r.buffer(f32, streams * groups);
+ defer best.deinit();
+ const at = try r.buffer(u32, streams * groups);
+ defer at.deinit();
+ const picks = try r.buffer(u32, 2 * streams);
+ defer picks.deinit();
+ const cb = r.queue.commandBuffer();
+ const e = cb.compute(.serial);
+ e.setPipeline(r.k.get("lfm2_markov_scan_bf16"));
+ for ([_]mtl.Buffer{ logits, w1, w2, ids }, 0..) |b, i| e.setBuffer(b, 0, i);
+ e.setValue([4]u32{ 2 * vocab, rank, vocab, 2 }, 4);
+ for ([_]mtl.Buffer{ best, at }, 5..) |b, i| e.setBuffer(b, 0, i);
+ e.dispatchGroups(mtl.Size.of(groups, streams, 1), mtl.Size.of(256, 1, 1));
+ e.setPipeline(r.k.get("lfm2_markov_pick"));
+ e.setBuffer(best, 0, 0);
+ e.setBuffer(at, 0, 1);
+ e.setValue(@as(u32, groups), 2);
+ e.setBuffer(picks, 0, 3);
+ e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1));
+ e.end();
+ cb.commit();
+ cb.wait();
+ const got = picks.slice(u32, 2 * streams);
+ const logit = logits.slice(u16, streams * 2 * vocab);
+ for (0..streams) |s| {
+ if (s == 3) {
+ try std.testing.expectEqual(@as(u32, 1), got[2 * s + 1]);
+ continue;
+ }
+ try std.testing.expectEqual(@as(u32, 0), got[2 * s + 1]);
+ const id = ids.slice(u32, 2 * streams)[2 * s];
+ var top: f64 = -std.math.inf(f64);
+ var mine: f64 = 0;
+ for (0..vocab) |v| {
+ var dot: f64 = 0;
+ for (0..rank) |i| {
+ const e1: f64 = if (id < vocab) quant.toF32(w1.slice(u16, vocab * rank)[id * rank + i]) else 0;
+ dot += @as(f64, quant.toF32(w2.slice(u16, vocab * rank)[v * rank + i])) * e1;
+ }
+ const score: f64 = quant.toF32(quant.toBf16(quant.toF32(logit[s * 2 * vocab + v]) + quant.toF32(quant.toBf16(@floatCast(dot)))));
+ top = @max(top, score);
+ if (v == got[2 * s]) mine = score;
+ }
+ try std.testing.expect(got[2 * s] < vocab and mine >= top - @abs(top) * 0x1p-7);
+ }
+}
diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig
new file mode 100644
index 0000000000..a737582c5e
--- /dev/null
+++ b/zig/src/families/lfm2_moe/dspark_block.zig
@@ -0,0 +1,445 @@
+//! DSpark's passes on the target's queue: context rows into a lane's keys, and a block's rows into Markov drafts.
+const std = @import("std");
+const mtl = @import("metal");
+const c = @import("config.zig");
+const st = @import("state.zig");
+const fwd = @import("forward.zig");
+const tile = @import("../../core/tile_qmm.zig");
+const am = @import("../../core/affine_mm.zig");
+const round = @import("tree_round_gpu");
+const ds = @import("dspark.zig");
+const dw = @import("dspark_weights.zig");
+const Drafter = ds.Drafter;
+const Lane = ds.Lane;
+
+const kw = c.kv_heads * c.head_dim;
+const QkArgs = extern struct { len: u32, cap: u32, eps: f32, log2_theta: f32 };
+const AttnArgs = extern struct { len: u32, cap: u32, scale: f32, subs: u32 };
+/// Partials a query row takes in the scratch (the kernels' QH x CHUNKS x SUBS).
+const part_row = c.query_heads * st.chunks * st.subs;
+
+fn attnArgs(lane: *const Lane, keys: u64) AttnArgs {
+ return .{ .len = @intCast(keys - 1), .cap = @intCast(lane.cap), .scale = 1.0 / @sqrt(@as(f32, c.head_dim)), .subs = st.subs };
+}
+const Pick = round.contract.Pick;
+const Bf16Args = extern struct { rows: u32, n: u32, k: u32 };
+const MarkovArgs = extern struct { stride: u32, rank: u32, vocab: u32, prev_stride: u32 };
+const ConfidenceArgs = extern struct { n: u32, pass: u32, vocab: u32, rank: u32, hidden: u32 };
+/// Tokens a Markov scan threadgroup takes (the kernel's MARKOV_SPAN), and threadgroups a stream's row takes.
+pub const scan_span = 512;
+pub const scan_groups = (c.vocab + scan_span - 1) / scan_span;
+
+/// Rows a drafter projection takes on the scalar twin at most (faster there than the window or tensor units).
+const scalar_rows = 8;
+/// Rows from which the target's head (128,000 outputs) runs on the tensor units, where the GPU has them.
+const head_rows = 5;
+
+/// Rows from which `w` runs on the tensor units: outputs of the hidden width or fewer stay on the twin to its rows.
+fn naxFrom(w: tile.Matrix) usize {
+ if (w.n <= c.hidden) return tile.twin_rows + 1;
+ return if (w.n == c.vocab) head_rows else scalar_rows + 1;
+}
+
+/// A buffer and byte offset.
+pub const Ref = struct { buf: mtl.Buffer, off: usize = 0 };
+
+/// One stream's block in a hold pass: rows [pending, mask x (block - 1)] at positions anchor .. anchor + block - 1.
+pub const Block = struct { lane: *Lane, pending: u32, anchor: u64, depth: u32 };
+
+const Enc = struct {
+ d: *Drafter,
+ e: mtl.ComputeEncoder,
+ bf16_rows_max: usize = 16,
+ conc: bool = false, // a concurrent encoder: each step waits for the writes before it at `sync`
+
+ /// Later dispatches see the writes before this point (a no-op on a serial encoder).
+ fn sync(x: Enc) void {
+ if (x.conc) x.e.barrier();
+ }
+
+ fn pipe(x: Enc, comptime key: []const u8) void {
+ x.e.setPipeline(x.d.target.kernels.get(key));
+ }
+
+ /// y[rows] = x[rows] W^T: the M5's tensor units from naxFrom rows, else the twin (scalar to scalar_rows) or tiles.
+ fn project(x: Enc, w: dw.Linear, in: Ref, out: Ref, rows: usize) !void {
+ return x.projectAll(&.{w}, in, &.{out}, rows);
+ }
+
+ /// y_i[rows] = x[rows] W_i^T on one input and shape: bf16 matrices straight, else one sums pass, products overlap.
+ fn projectAll(x: Enc, lins: []const dw.Linear, in: Ref, outs: []const Ref, rows: usize) !void {
+ if (lins[0] == .bf16) {
+ for (lins, outs) |lin, out| x.bf16(lin.bf16, in, out, rows);
+ return;
+ }
+ var buf: [4]tile.Matrix = undefined;
+ for (lins, 0..) |lin, i| buf[i] = lin.codes;
+ const ws = buf[0..lins.len];
+ const t = &x.d.target.kernels.tiles;
+ const sums = Ref{ .buf = x.d.s.sums };
+ const k = ws[0].k;
+ const nax = x.d.target.kernels.naxPipes(ws[0].bits);
+ if (rows >= naxFrom(ws[0])) if (nax) |p| {
+ am.rowSums(x.e, p, c.group, in, sums, @intCast(rows), @intCast(k));
+ x.sync();
+ for (ws, outs) |w, out| {
+ const q = .{ .w = Ref{ .buf = w.w, .off = w.w_off }, .s = Ref{ .buf = w.scales, .off = w.s_off }, .b = Ref{ .buf = w.biases, .off = w.b_off }, .n = @as(u32, @intCast(w.n)), .k = @as(u32, @intCast(w.k)) };
+ am.dense(x.e, p, in, sums, q, out, @intCast(rows));
+ }
+ return;
+ };
+ if (rows <= tile.twin_rows and x.d.target.kernels.twin) {
+ // beside the tensor units the scalar twin takes every row count it has; elsewhere the window from 9 rows
+ const window: usize = if (nax != null) tile.twin_rows + 1 else scalar_rows + 1;
+ for (ws, outs) |w, out| try tile.encodeTwinFrom(x.e, t, w, in.buf, in.off, out.buf, out.off, rows, window);
+ return;
+ }
+ tile.encodeSums(x.e, t, k, in.buf, in.off, x.d.s.sums, rows);
+ x.sync();
+ for (ws, outs) |w, out| try tile.encode(x.e, t, w, in.buf, in.off, x.d.s.sums, 0, out.buf, out.off, rows);
+ }
+
+ /// y[rows] = x[rows] W^T over the released bf16 matrix: one 16-row pass of W, else 64x64 tiles.
+ fn bf16(x: Enc, m: dw.Bf16, in: Ref, out: Ref, rows: usize) void {
+ const args = Bf16Args{ .rows = @intCast(rows), .n = @intCast(m.n), .k = @intCast(m.k) };
+ const few = rows <= x.bf16_rows_max;
+ if (few) x.pipe("lfm2_bf16_rows") else x.pipe("lfm2_bf16_mma");
+ x.e.setBuffer(in.buf, in.off, 0);
+ x.e.setBuffer(m.w, 0, 1);
+ x.e.setBytes(std.mem.asBytes(&args), 2);
+ x.e.setBuffer(out.buf, out.off, 3);
+ if (few) {
+ x.e.dispatchGroups(mtl.Size.of((m.n + 7) / 8, (rows + 15) / 16, 1), mtl.Size.of(256, 1, 1));
+ } else x.e.dispatchGroups(mtl.Size.of((m.n + 63) / 64, (rows + 63) / 64, 1), mtl.Size.of(128, 1, 1));
+ }
+
+ /// h += r when `add`, then out = norm(h) * gain: LFM2's own norm (the drafter's width and eps are the target's).
+ fn norm(x: Enc, h: mtl.Buffer, r: mtl.Buffer, gain: mtl.Buffer, add: bool, out: mtl.Buffer, rows: usize) void {
+ x.pipe("lfm2_norm");
+ x.e.setBuffer(h, 0, 0);
+ x.e.setBuffer(r, 0, 1);
+ x.e.setBuffer(gain, 0, 2);
+ x.e.setValue(c.eps, 3);
+ x.e.setValue(@as(u32, @intFromBool(add)), 4);
+ x.e.setBuffer(out, 0, 5);
+ x.e.dispatchGroups(mtl.Size.of(rows, 1, 1), mtl.Size.of(c.hidden / 4, 1, 1));
+ }
+
+ /// Rows' queries normed and rotated into `queries`, keys and values into the lane's layer at positions from `len`.
+ fn qk(x: Enc, q: Ref, k: Ref, v: Ref, layer: usize, lane: *const Lane, len: u64, queries: Ref, rows: usize) void {
+ const ly = &x.d.w.layers[layer];
+ x.pipe("lfm2_qk");
+ x.e.setBuffer(q.buf, q.off, 0);
+ x.e.setBuffer(k.buf, k.off, 1);
+ x.e.setBuffer(v.buf, v.off, 2);
+ x.e.setBuffer(ly.q_norm, 0, 3);
+ x.e.setBuffer(ly.k_norm, 0, 4);
+ const args = QkArgs{ .len = @intCast(len), .cap = @intCast(lane.cap), .eps = c.eps, .log2_theta = std.math.log2(x.d.config.theta) };
+ x.e.setBytes(std.mem.asBytes(&args), 5);
+ x.e.setBuffer(queries.buf, queries.off, 6);
+ x.e.setBuffer(lane.mem, lane.kv(layer, false).off, 7);
+ x.e.setBuffer(lane.mem, lane.kv(layer, true).off, 8);
+ x.e.dispatchGroups(mtl.Size.of(c.query_heads + c.kv_heads, rows, 1), mtl.Size.of(32, 1, 1));
+ }
+
+ /// The chunks' partials of query rows [row, row + rows) against the lane's keys, at partial row `part`.
+ fn partials(x: Enc, layer: usize, lane: *const Lane, row: usize, rows: usize, keys: u64, part: usize) void {
+ const s = &x.d.s;
+ const args = attnArgs(lane, keys);
+ const used = (keys + st.chunk_keys - 1) / st.chunk_keys * st.subs;
+ const at = part * part_row;
+ x.pipe("lfm2_attn_block");
+ x.e.setBuffer(s.queries, row * c.hidden * 2, 0);
+ x.e.setBuffer(lane.mem, lane.kv(layer, false).off, 1);
+ x.e.setBuffer(lane.mem, lane.kv(layer, true).off, 2);
+ x.e.setBytes(std.mem.asBytes(&args), 3);
+ x.e.setBuffer(s.pm, at * 4, 4);
+ x.e.setBuffer(s.pl, at * 4, 5);
+ x.e.setBuffer(s.po, at * c.head_dim * 4, 6);
+ x.e.setValue(@as(u32, @intCast(rows)), 7);
+ x.e.dispatchGroups(mtl.Size.of((rows + 7) / 8, c.kv_heads, @intCast(used)), mtl.Size.of(128, 1, 1));
+ }
+
+ /// The partials from partial row `part` merged into the attention's output rows [row, row + rows).
+ fn merge(x: Enc, lane: *const Lane, row: usize, rows: usize, keys: u64, part: usize) void {
+ const s = &x.d.s;
+ const args = attnArgs(lane, keys);
+ const at = part * part_row;
+ x.pipe("lfm2_attn_block_merge");
+ x.e.setBuffer(s.pm, at * 4, 0);
+ x.e.setBuffer(s.pl, at * 4, 1);
+ x.e.setBuffer(s.po, at * c.head_dim * 4, 2);
+ x.e.setBytes(std.mem.asBytes(&args), 3);
+ x.e.setBuffer(s.y, row * c.hidden * 2, 4);
+ x.e.dispatchGroups(mtl.Size.of(c.query_heads, rows, 1), mtl.Size.of(32, 1, 1));
+ }
+};
+
+/// One stream's tapped rows in an absorb pass: `rows` rows at `src`, the context from position `p0`.
+pub const Piece = struct { lane: *Lane, src: Ref, rows: usize, p0: u64 };
+
+/// Each piece's rows into its lane's keys and values (fc, hidden_norm, k, v, k_norm, RoPE) in passes on `e`.
+pub fn absorbPass(d: *Drafter, e: mtl.ComputeEncoder, pieces: []const Piece) !void {
+ return absorbOn(.{ .d = d, .e = e }, pieces);
+}
+
+fn absorbOn(encoder: Enc, pieces: []const Piece) !void {
+ var x = encoder;
+ x.bf16_rows_max = 32; // preserve the context cache's original projection arithmetic
+ const d = x.d;
+ const s = &d.s;
+ var rows: usize = 0;
+ for (pieces) |pc| {
+ try x.project(d.w.fc, pc.src, .{ .buf = s.fcx, .off = rows * c.hidden * 2 }, pc.rows);
+ x.sync(); // the pieces' projections share the sums scratch
+ rows += pc.rows;
+ }
+ if (rows == 0 or rows > ds.absorb_rows) return error.DraftContextGap;
+ x.norm(s.fcx, s.fcx, d.w.hidden_norm, false, s.fcx, rows); // in place: a thread reads its values first
+ x.sync();
+ for (0..d.config.layers) |i| {
+ const ly = &d.w.layers[i];
+ try x.projectAll(&.{ ly.k, ly.v }, .{ .buf = s.fcx }, &.{ .{ .buf = s.k }, .{ .buf = s.v } }, rows);
+ x.sync();
+ var at: usize = 0;
+ for (pieces) |pc| {
+ const q = Ref{ .buf = s.fcx, .off = at * c.hidden * 2 };
+ x.qk(q, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, i, pc.lane, pc.p0, .{ .buf = s.dump }, pc.rows);
+ at += pc.rows;
+ }
+ x.sync();
+ }
+}
+
+/// A command buffer for absorb passes, and its wait.
+pub fn begin(d: *Drafter) struct { cb: mtl.CommandBuffer, e: mtl.ComputeEncoder } {
+ const cb = d.target.queue.commandBuffer();
+ return .{ .cb = cb, .e = cb.compute(.serial) };
+}
+
+pub fn end(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void {
+ e.end();
+ cb.commit();
+ cb.wait();
+ if (cb.failure()) |msg| {
+ std.log.err("lfm2 dspark: command buffer failed: {s}", .{msg});
+ return error.GpuFailed;
+ }
+}
+
+/// Layer `layer`'s q, k, v and every block's attention (a lone block's last layer: `out` rows), on a concurrent `x`.
+fn attendBlocks(x: Enc, layer: usize, blocks: []const Block, n: u32, rows: usize, out: usize) !void {
+ const s = &x.d.s;
+ const ly = &x.d.w.layers[layer];
+ // the three projections, every block's keys, then waves of as many blocks as the partials hold at once
+ try x.projectAll(&.{ ly.q, ly.k, ly.v }, .{ .buf = s.x }, &.{ .{ .buf = s.q }, .{ .buf = s.k }, .{ .buf = s.v } }, rows);
+ x.sync();
+ for (blocks, 0..) |b, bi| {
+ const at = bi * n;
+ x.qk(.{ .buf = s.q, .off = at * c.hidden * 2 }, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, layer, b.lane, b.anchor, .{ .buf = s.queries, .off = at * c.hidden * 2 }, n);
+ }
+ x.sync();
+ const each = if (blocks.len == 1) out else n;
+ const wave = @max(1, x.d.partRows() / n);
+ var first: usize = 0;
+ while (first < blocks.len) : (first += wave) {
+ const last = @min(blocks.len, first + wave);
+ for (blocks[first..last], first..) |b, bi| x.partials(layer, b.lane, bi * n, each, b.anchor + n, (bi - first) * n);
+ x.sync();
+ for (blocks[first..last], first..) |b, bi| x.merge(b.lane, bi * n, each, b.anchor + n, (bi - first) * n);
+ x.sync();
+ }
+}
+
+/// A GPU event a hold signals instead of being waited on, and the value it sets.
+pub const Signal = struct { event: mtl.Event, value: u64 };
+
+/// The blocks' drafts (backbone, then Markov chain) into the lanes' held chains; with `signal`, left for readPicks.
+pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBuffer {
+ const pool = mtl.objc.Pool.push();
+ defer pool.pop();
+ const s = &d.s;
+ var depth: u32 = 0;
+ for (blocks) |b| depth = @max(depth, b.depth);
+ // a block of the deepest chain's rows, at most the 9 the drafter was trained on
+ const n: u32 = depth;
+ const rows = blocks.len * n;
+ if (rows == 0 or rows > ds.pass_rows) return error.DraftTooManyStreams;
+ const ids = s.ids.slice(u32, ds.pass_rows);
+ for (0..blocks.len) |i| {
+ ids[i * n] = blocks[i].pending;
+ @memset(ids[i * n + 1 ..][0 .. n - 1], d.config.mask);
+ }
+ const steps = depth;
+ // past the last layer's attention a lone block carries only the rows its chain reads; several keep them all
+ const tail = if (blocks.len == 1) steps else rows;
+ const cb = d.target.queue.commandBuffer();
+ // one concurrent encoder: independent dispatches overlap, each dependent step waits at a barrier
+ const x = Enc{ .d = d, .e = cb.compute(.concurrent), .conc = true };
+ // the blocks' staged context rows first, in the same command buffer
+ var staged: [ds.pass_rows]Piece = undefined;
+ var count: usize = 0;
+ var held: usize = 0;
+ for (blocks) |b| if (ds.Drafter.stagedPiece(b.lane)) |pc| {
+ if (held + pc.rows > ds.absorb_rows) {
+ try absorbOn(x, staged[0..count]);
+ count = 0;
+ held = 0;
+ }
+ staged[count] = pc;
+ count += 1;
+ held += pc.rows;
+ };
+ if (count > 0) try absorbOn(x, staged[0..count]);
+ for (blocks) |b| {
+ b.lane.end += b.lane.staged;
+ b.lane.staged = 0;
+ }
+ const t = d.target;
+ const emb = t.weights.embedding;
+ x.e.setPipeline(t.kernels.embed(emb.bits));
+ x.e.setBuffer(s.ids, 0, 0);
+ x.e.setBuffer(emb.w, emb.w_off, 1);
+ x.e.setBuffer(emb.scales, emb.s_off, 2);
+ x.e.setBuffer(emb.biases, emb.b_off, 3);
+ x.e.setBuffer(s.h, 0, 4);
+ const units = c.hidden / @as(usize, if (emb.bits == 6) 4 else 8 / emb.bits);
+ x.e.dispatchThreads(mtl.Size.of(units, rows, 1), mtl.Size.of(256, 1, 1));
+ x.sync();
+ for (0..d.config.layers) |i| {
+ const ly = &d.w.layers[i];
+ const out = if (i + 1 == d.config.layers) tail else rows;
+ x.norm(s.h, s.r, ly.input_norm, i > 0, s.x, rows);
+ x.sync();
+ try attendBlocks(x, i, blocks, n, rows, out);
+ try x.project(ly.o, .{ .buf = s.y }, .{ .buf = s.r }, out);
+ x.sync();
+ x.norm(s.h, s.r, ly.post_norm, true, s.x, out);
+ x.sync();
+ try x.projectAll(&.{ ly.gate, ly.up }, .{ .buf = s.x }, &.{ .{ .buf = s.gate }, .{ .buf = s.up } }, out);
+ x.sync();
+ x.pipe("lfm2_swiglu");
+ x.e.setBuffer(s.gate, 0, 0);
+ x.e.setBuffer(s.up, 0, 1);
+ x.e.setBuffer(s.act, 0, 2);
+ x.e.setValue(@as(u32, @intCast(out * d.config.intermediate)), 3);
+ x.e.dispatchThreads(mtl.Size.of(out * d.config.intermediate / 8, 1, 1), mtl.Size.of(256, 1, 1));
+ x.sync();
+ try x.project(ly.down, .{ .buf = s.act }, .{ .buf = s.r }, out);
+ x.sync();
+ }
+ x.norm(s.h, s.r, d.w.norm, true, s.x, tail);
+ x.sync();
+ try x.project(.{ .codes = fwd.matrix(emb) }, .{ .buf = s.x }, .{ .buf = s.logits }, tail);
+ x.sync();
+ try markov(x, blocks.len, n, steps);
+ if (d.confident()) confidence(x, blocks.len, n, steps);
+ const e = x.e;
+ if (signal) |sg| { // committed unwaited, the command buffer returned retained
+ e.end();
+ cb.signalEvent(sg.event, sg.value);
+ const kept = cb.retain();
+ cb.commit();
+ return kept;
+ }
+ try end(cb, e);
+ const picks = s.picks.slice(Pick, d.config.block * ds.pass_rows);
+ for (blocks, 0..) |b, bi| {
+ for (b.lane.held[0..steps], 0..) |*h, i| {
+ const p = picks[i * ds.pass_rows + bi];
+ if (p.nonfinite != 0 or p.token >= c.vocab) return error.DraftNonfinite;
+ h.* = p.token;
+ }
+ b.lane.held_len = b.depth;
+ }
+ for (blocks, 0..) |b, bi| readConfidence(d, b.lane, b.depth, bi);
+ return null;
+}
+
+/// Lane `l`'s drafts' chances from slot `bi` of the last hold's logits: sigmoid(logit + bias); none: left as they are.
+fn readConfidence(d: *Drafter, l: *Lane, depth: u32, bi: usize) void {
+ if (!d.confident()) return;
+ const logits = d.s.conf.slice(f32, d.config.block * ds.pass_rows);
+ for (l.conf[0..depth], 0..) |*p, i| p.* = 1 / (1 + @exp(-(logits[i * ds.pass_rows + bi] + d.w.confidence_bias)));
+ l.conf_len = depth;
+}
+
+/// A finished hold's picks into the lanes' held chains; a nonfinite or invalid pick is refused, as when synchronous.
+pub fn readPicks(d: *Drafter, lanes: []const *Lane, depths: []const u32) !void {
+ const picks = d.s.picks.slice(Pick, d.config.block * ds.pass_rows);
+ var chains: [ds.pass_rows][]u32 = undefined;
+ for (lanes, chains[0..lanes.len]) |l, *ch| ch.* = &l.held;
+ try copyPicks(picks, chains[0..lanes.len], depths);
+ for (lanes, depths, 0..) |l, depth, bi| {
+ l.held_len = depth;
+ readConfidence(d, l, depth, bi);
+ }
+}
+
+/// Step i of block bi is picks[i * ds.pass_rows + bi]: each block's first `depth` picks into its chain.
+pub fn copyPicks(picks: []const Pick, chains: []const []u32, depths: []const u32) !void {
+ for (chains, depths, 0..) |chain, depth, bi| {
+ for (chain[0..depth], 0..) |*h, i| {
+ const p = picks[i * ds.pass_rows + bi];
+ if (p.nonfinite != 0 or p.token >= c.vocab) return error.DraftNonfinite;
+ h.* = p.token;
+ }
+ }
+}
+
+/// Each stream's draft i confidence logit (the final norm's row i, W1 of the token before draft i), for readConfidence.
+fn confidence(x: Enc, streams: usize, n: u32, depth: u32) void {
+ const d = x.d;
+ const e = x.e;
+ x.pipe("lfm2_dspark_confidence");
+ e.setBuffer(d.s.x, 0, 0);
+ e.setBuffer(d.w.w1, 0, 1);
+ e.setBuffer(d.s.ids, 0, 2);
+ e.setBuffer(d.s.picks, 0, 3);
+ e.setBuffer(d.w.confidence.?, 0, 4);
+ const args = ConfidenceArgs{ .n = n, .pass = ds.pass_rows, .vocab = c.vocab, .rank = d.config.rank, .hidden = c.hidden };
+ e.setBytes(std.mem.asBytes(&args), 5);
+ e.setBuffer(d.s.conf, 0, 6);
+ e.dispatchGroups(mtl.Size.of(depth, streams, 1), mtl.Size.of(256, 1, 1));
+}
+
+/// Step i of each stream: its row i's logits plus W2 W1[previous draft] (the pending token first), then the argmax.
+fn markov(x: Enc, streams: usize, n: u32, depth: u32) !void {
+ const d = x.d;
+ const s = &d.s;
+ const e = x.e;
+ for (0..depth) |i| {
+ // step i's previous token: the pending one (each block's first id), then step i - 1's pick
+ switch (d.w.w2) {
+ .codes => x.pipe("lfm2_markov_scan"),
+ .bf16 => x.pipe("lfm2_markov_scan_bf16"),
+ }
+ e.setBuffer(s.logits, i * c.vocab * 2, 0);
+ e.setBuffer(d.w.w1, 0, 1);
+ e.setBuffer(switch (d.w.w2) {
+ .codes => |m| m.w,
+ .bf16 => |m| m.w,
+ }, switch (d.w.w2) {
+ .codes => |m| m.w_off,
+ .bf16 => 0,
+ }, 2);
+ if (i == 0) e.setBuffer(s.ids, 0, 3) else e.setBuffer(s.picks, (i - 1) * ds.pass_rows * @sizeOf(Pick), 3);
+ const args = MarkovArgs{ .stride = n * c.vocab, .rank = d.config.rank, .vocab = c.vocab, .prev_stride = if (i == 0) n else @sizeOf(Pick) / 4 };
+ e.setBytes(std.mem.asBytes(&args), 4);
+ e.setBuffer(s.best, 0, 5);
+ e.setBuffer(s.at, 0, 6);
+ if (d.w.w2 == .codes) {
+ e.setBuffer(d.w.w2.codes.scales, d.w.w2.codes.s_off, 7);
+ e.setBuffer(d.w.w2.codes.biases, d.w.w2.codes.b_off, 8);
+ }
+ e.dispatchGroups(mtl.Size.of(scan_groups, streams, 1), mtl.Size.of(256, 1, 1));
+ x.sync();
+ x.pipe("lfm2_markov_pick");
+ e.setBuffer(s.best, 0, 0);
+ e.setBuffer(s.at, 0, 1);
+ e.setValue(@as(u32, scan_groups), 2);
+ e.setBuffer(s.picks, i * ds.pass_rows * @sizeOf(Pick), 3);
+ e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1));
+ x.sync();
+ }
+}
diff --git a/zig/src/families/lfm2_moe/dspark_config.zig b/zig/src/families/lfm2_moe/dspark_config.zig
new file mode 100644
index 0000000000..68acc6d30d
--- /dev/null
+++ b/zig/src/families/lfm2_moe/dspark_config.zig
@@ -0,0 +1,142 @@
+//! LiquidAI's DSpark drafter for LFM2.5-24B-A2B: its config admitted, any other shape refused with its reason.
+const std = @import("std");
+const c = @import("config.zig");
+
+/// The geometry the drafter's kernels take: the target's attention shape, five taps at most eight layers.
+pub const Config = struct {
+ taps: [8]u32 = @splat(0),
+ tap_count: u32,
+ layers: u32,
+ intermediate: u32,
+ /// Rows a block takes: the pending token and block - 1 mask tokens, one draft a row.
+ block: u32,
+ mask: u32,
+ rank: u32,
+ theta: f32,
+
+ pub fn read(gpa: std.mem.Allocator, io: std.Io, dir: []const u8) !Config {
+ const path = try std.fs.path.join(gpa, &.{ dir, "config.json" });
+ defer gpa.free(path);
+ const bytes = try std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(1 << 20));
+ defer gpa.free(bytes);
+ return parse(gpa, bytes);
+ }
+
+ pub fn parse(gpa: std.mem.Allocator, bytes: []const u8) !Config {
+ var parsed = try std.json.parseFromSlice(std.json.Value, gpa, bytes, .{});
+ defer parsed.deinit();
+ const o = if (parsed.value == .object) parsed.value.object else return error.BadDraftConfig;
+ const archs = o.get("architectures") orelse return error.BadDraftConfig;
+ if (archs != .array or archs.array.items.len != 1 or archs.array.items[0] != .string) return error.BadDraftConfig;
+ if (!std.mem.eql(u8, archs.array.items[0].string, "Lfm2DSparkDraftModel")) return error.NotLfmDSpark;
+ const d = if (o.get("dflash_config")) |v| (if (v == .object) v.object else return error.BadDraftConfig) else return error.BadDraftConfig;
+ var out: Config = .{
+ .tap_count = 0,
+ .layers = try small(o, "num_hidden_layers"),
+ .intermediate = try small(o, "intermediate_size"),
+ .block = try small(o, "block_size"),
+ .mask = try small(d, "mask_token_id"),
+ .rank = try small(o, "markov_rank"),
+ .theta = @floatCast(try number(o, "rope_theta")),
+ };
+ // the target's widths: the drafter shares its embedding, head and attention kernels
+ if (try small(o, "hidden_size") != c.hidden or try small(o, "vocab_size") != c.vocab) return error.DraftForAnotherTarget;
+ if (try small(d, "num_target_layers") != c.layers) return error.DraftForAnotherTarget;
+ const heads = try small(o, "num_attention_heads");
+ const kv = try small(o, "num_key_value_heads");
+ if (heads != c.query_heads or kv != c.kv_heads or try small(o, "head_dim") != c.head_dim) return error.UnsupportedDraftShape;
+ if (@as(f32, @floatCast(try number(o, "rms_norm_eps"))) != c.eps) return error.UnsupportedDraftShape;
+ const taps = d.get("target_layer_ids") orelse return error.BadDraftConfig;
+ if (taps != .array or taps.array.items.len == 0 or taps.array.items.len > out.taps.len) return error.BadDraftConfig;
+ for (taps.array.items, 0..) |t, i| {
+ if (t != .integer or t.integer < 0) return error.BadDraftConfig;
+ if (t.integer >= c.layers) return error.DraftForAnotherTarget;
+ out.taps[i] = @intCast(t.integer);
+ }
+ out.tap_count = @intCast(taps.array.items.len);
+ try kinds(o, out.layers);
+ // the block's MLP is the fixed SiLU gate
+ const act = o.get("hidden_act") orelse return error.BadDraftConfig;
+ if (act != .string or !std.mem.eql(u8, act.string, "silu")) return error.UnsupportedDraftActivation;
+ // the vendored defaults decide the block: non-causal attention, interleaved RoPE, a plain Markov head
+ if (o.get("is_causal") != null) return error.UnsupportedDraftMask;
+ const neox = o.get("rope_is_neox_style") orelse return error.BadDraftConfig;
+ if (neox != .bool or neox.bool) return error.UnsupportedDraftRope;
+ const head = o.get("markov_head_type") orelse return error.BadDraftConfig;
+ if (head != .string or !std.mem.eql(u8, head.string, "vanilla")) return error.UnsupportedMarkovHead;
+ if (o.get("sample_from_anchor")) |v| if (v != .bool or !v.bool) return error.UnsupportedDraftAnchor;
+ if (out.layers == 0 or out.layers > 8 or out.block < 2 or out.block > 16) return error.UnsupportedDraftShape;
+ if (out.intermediate == 0 or out.intermediate % 64 != 0 or out.rank == 0 or out.rank > 256 or out.rank % 64 != 0) return error.UnsupportedDraftShape;
+ if (out.mask >= c.vocab or !std.math.isFinite(out.theta) or !(out.theta > 0)) return error.BadDraftConfig;
+ return out;
+ }
+};
+
+/// Every layer full attention (the block's mask has no sliding window).
+fn kinds(o: std.json.ObjectMap, layers: u32) !void {
+ const v = o.get("layer_types") orelse return;
+ if (v != .array or v.array.items.len != layers) return error.BadDraftConfig;
+ for (v.array.items) |k| if (k != .string or !std.mem.eql(u8, k.string, "full_attention")) return error.UnsupportedDraftMask;
+}
+
+fn small(o: std.json.ObjectMap, key: []const u8) !u32 {
+ const v = o.get(key) orelse return error.BadDraftConfig;
+ if (v != .integer or v.integer < 0 or v.integer > std.math.maxInt(u32)) return error.BadDraftConfig;
+ return @intCast(v.integer);
+}
+
+fn number(o: std.json.ObjectMap, key: []const u8) !f64 {
+ const v = o.get(key) orelse return error.BadDraftConfig;
+ return switch (v) {
+ .integer => |i| @floatFromInt(i),
+ .float => |f| f,
+ else => error.BadDraftConfig,
+ };
+}
+
+/// LiquidAI/LFM2.5-24B-A2B-DSpark's config.json at e052e5b1dd10, as released.
+pub const stock =
+ \\{"architectures":["Lfm2DSparkDraftModel"],"model_type":"qwen3","dtype":"bfloat16","hidden_size":2048,
+ \\"num_hidden_layers":5,"num_attention_heads":32,"num_key_value_heads":8,"head_dim":64,"intermediate_size":6144,
+ \\"hidden_act":"silu","rms_norm_eps":1e-05,"vocab_size":128000,"rope_theta":10000000.0,
+ \\"max_position_embeddings":131072,"layer_types":["full_attention","full_attention","full_attention",
+ \\"full_attention","full_attention"],"block_size":9,"dflash_config":{"mask_token_id":125017,
+ \\"target_layer_ids":[2,10,18,26,34],"num_target_layers":40},"markov_rank":256,"rope_is_neox_style":false,
+ \\"enable_confidence_head":true,"markov_head_type":"vanilla"}
+;
+
+fn edited(from: []const u8, to: []const u8) !Config {
+ const a = std.testing.allocator;
+ const text = try std.mem.replaceOwned(u8, a, stock, from, to);
+ defer a.free(text);
+ return Config.parse(a, text);
+}
+
+test "lfm2 dspark: the released config is admitted with its taps, block, mask token and Markov rank" {
+ const cfg = try Config.parse(std.testing.allocator, stock);
+ try std.testing.expectEqualSlices(u32, &.{ 2, 10, 18, 26, 34 }, cfg.taps[0..cfg.tap_count]);
+ try std.testing.expectEqual(@as(u32, 5), cfg.layers);
+ try std.testing.expectEqual(@as(u32, 9), cfg.block);
+ try std.testing.expectEqual(@as(u32, 125017), cfg.mask);
+ try std.testing.expectEqual(@as(u32, 256), cfg.rank);
+ try std.testing.expectEqual(@as(u32, 6144), cfg.intermediate);
+ try std.testing.expectEqual(@as(f32, 1e7), cfg.theta);
+}
+
+test "lfm2 dspark: other drafters and shapes are refused with their reason" {
+ try std.testing.expectError(error.NotLfmDSpark, edited("Lfm2DSparkDraftModel", "DFlashDraftModel"));
+ try std.testing.expectError(error.DraftForAnotherTarget, edited("\"hidden_size\":2048", "\"hidden_size\":4096"));
+ try std.testing.expectError(error.DraftForAnotherTarget, edited("\"num_target_layers\":40", "\"num_target_layers\":24"));
+ try std.testing.expectError(error.DraftForAnotherTarget, edited("26,34]", "26,44]"));
+ try std.testing.expectError(error.UnsupportedDraftShape, edited("\"head_dim\":64", "\"head_dim\":128"));
+ try std.testing.expectError(error.UnsupportedDraftActivation, edited("\"silu\"", "\"gelu\""));
+ try std.testing.expectError(error.UnsupportedDraftRope, edited("\"rope_is_neox_style\":false", "\"rope_is_neox_style\":true"));
+ try std.testing.expectError(error.UnsupportedMarkovHead, edited("\"vanilla\"", "\"gated\""));
+ try std.testing.expectError(error.UnsupportedDraftMask, edited("\"full_attention\"]", "\"sliding_attention\"]"));
+ try std.testing.expectError(error.UnsupportedDraftMask, edited("\"block_size\"", "\"is_causal\":true,\"block_size\""));
+ try std.testing.expectError(error.UnsupportedDraftAnchor, edited("\"block_size\"", "\"sample_from_anchor\":false,\"block_size\""));
+ try std.testing.expectError(error.UnsupportedDraftShape, edited("\"block_size\":9", "\"block_size\":17"));
+ try std.testing.expectError(error.UnsupportedDraftShape, edited("\"markov_rank\":256", "\"markov_rank\":512"));
+ try std.testing.expectError(error.BadDraftConfig, edited("10000000.0", "1e100"));
+ try std.testing.expectError(error.BadDraftConfig, edited("\"mask_token_id\":125017", "\"mask_token_id\":128000"));
+}
diff --git a/zig/src/families/lfm2_moe/dspark_quant.zig b/zig/src/families/lfm2_moe/dspark_quant.zig
new file mode 100644
index 0000000000..a34b52dd20
--- /dev/null
+++ b/zig/src/families/lfm2_moe/dspark_quant.zig
@@ -0,0 +1,118 @@
+//! BF16 drafter rows quantized at load to MLX's affine codes in groups of 64, the layout core/tile_qmm reads.
+const std = @import("std");
+
+/// MLX's affine codes of 64 bf16 values at `bits`: bias the larger edge, scale |edge| / round(|edge| / step).
+pub fn group(x: []const u16, bits: u8, codes: []u8, scale: *u16, bias: *u16) void {
+ var lo: f32 = std.math.inf(f32);
+ var hi: f32 = -std.math.inf(f32);
+ for (x) |v| {
+ lo = @min(lo, toF32(v));
+ hi = @max(hi, toF32(v));
+ }
+ const bins: f32 = @floatFromInt((@as(u32, 1) << @intCast(bits)) - 1);
+ const step = @max(hi - lo, 1e-7) / bins;
+ const from_min = @abs(lo) > @abs(hi);
+ const edge = if (from_min) lo else hi;
+ const q0 = @round(@abs(edge) / step);
+ const s_f: f32 = if (q0 == 0) step else (if (from_min) @abs(edge) else -@abs(edge)) / q0;
+ scale.* = toBf16(s_f);
+ bias.* = toBf16(if (q0 == 0) lo else edge);
+ const s = toF32(scale.*);
+ const b = toF32(bias.*);
+ @memset(codes, 0);
+ for (x, 0..) |v, i| {
+ const q: u8 = @intFromFloat(std.math.clamp(@round((toF32(v) - b) / s), 0, bins));
+ if (bits == 8) codes[i] = q else codes[i / 2] |= q << @intCast((i % 2) * 4);
+ }
+}
+
+pub fn toF32(v: u16) f32 {
+ return @bitCast(@as(u32, v) << 16);
+}
+
+/// Round to nearest even, as MLX casts.
+pub fn toBf16(f: f32) u16 {
+ const b: u32 = @bitCast(f);
+ if ((b & 0x7f80_0000) == 0x7f80_0000) return @truncate(b >> 16);
+ return @truncate((b + 0x7fff + ((b >> 16) & 1)) >> 16);
+}
+
+/// A matrix's codes [n][k * bits / 8], scales and biases [n][k / 64], each written by one thread's rows.
+pub const Job = struct {
+ src: []const u16,
+ k: usize,
+ bits: u8,
+ codes: []u8,
+ scales: []u16,
+ biases: []u16,
+ fn run(j: Job, from: usize, to: usize) void {
+ const groups = j.k / 64;
+ const per = 64 * @as(usize, j.bits) / 8;
+ for (from..to) |r| for (0..groups) |g| {
+ const at = r * groups + g;
+ group(j.src[r * j.k + g * 64 ..][0..64], j.bits, j.codes[at * per ..][0..per], &j.scales[at], &j.biases[at]);
+ };
+ }
+
+ /// The job over its rows on every CPU (at most 16 threads).
+ pub fn all(j: Job, gpa: std.mem.Allocator, rows: usize) !void {
+ const threads = @min(std.Thread.getCpuCount() catch 1, 16);
+ const per = (rows + threads - 1) / threads;
+ const handles = try gpa.alloc(std.Thread, threads);
+ defer gpa.free(handles);
+ var spawned: usize = 0;
+ defer for (handles[0..spawned]) |h| h.join();
+ var at: usize = 0;
+ while (at < rows) : (at += per) {
+ handles[spawned] = try std.Thread.spawn(.{}, run, .{ j, at, @min(rows, at + per) });
+ spawned += 1;
+ }
+ }
+};
+
+/// Head `h`'s interleaved RoPE pairs (2i, 2i + 1) as half-split pairs (i, i + dim / 2): row h * dim + j's source.
+pub fn halfSplit(row: usize, dim: usize) usize {
+ const h = row / dim;
+ const j = row % dim;
+ const half = dim / 2;
+ return h * dim + if (j < half) 2 * j else 2 * (j - half) + 1;
+}
+
+test "lfm2 dspark: 4- and 8-bit codes keep the larger edge exact, as MLX quantizes" {
+ var x: [64]u16 = undefined;
+ for (&x, 0..) |*v, i| v.* = toBf16(@as(f32, @floatFromInt(i)) * 0.125 - 2.0);
+ for ([_]u8{ 4, 8 }) |bits| {
+ var codes: [64]u8 = undefined;
+ var s: u16 = undefined;
+ var b: u16 = undefined;
+ group(&x, bits, codes[0 .. 64 * @as(usize, bits) / 8], &s, &b);
+ try std.testing.expect(toF32(s) < 0 and toF32(b) == 5.875); // the max is the edge: codes descend from it
+ const last: u8 = if (bits == 8) codes[63] else codes[31] >> 4;
+ try std.testing.expectEqual(@as(u8, 0), last);
+ for (x, 0..) |v, i| {
+ const q: u8 = if (bits == 8) codes[i] else (codes[i / 2] >> @intCast((i % 2) * 4)) & 0xf;
+ const back = toF32(s) * @as(f32, @floatFromInt(q)) + toF32(b);
+ try std.testing.expect(@abs(back - toF32(v)) <= @abs(toF32(s)) * 0.51);
+ }
+ }
+}
+
+test "lfm2 dspark: half-split rows rotated by the half-split rule equal interleaved pairs rotated, row for row" {
+ const dim = 64;
+ var x: [2 * dim]f32 = undefined;
+ for (&x, 0..) |*v, i| v.* = @sin(@as(f32, @floatFromInt(i)) * 0.37) * 3;
+ var moved: [2 * dim]f32 = undefined;
+ for (&moved, 0..) |*v, r| v.* = x[halfSplit(r, dim)];
+ const pos: f32 = 1234;
+ for (0..2) |h| for (0..dim / 2) |i| {
+ const angle = pos * std.math.pow(f32, 1e7, -@as(f32, @floatFromInt(i)) / (dim / 2));
+ const cs = @cos(angle);
+ const sn = @sin(angle);
+ const a = x[h * dim + 2 * i];
+ const b = x[h * dim + 2 * i + 1];
+ const l = moved[h * dim + i];
+ const r = moved[h * dim + i + dim / 2];
+ try std.testing.expectEqual(a * cs - b * sn, l * cs - r * sn);
+ try std.testing.expectEqual(a * sn + b * cs, l * sn + r * cs);
+ };
+}
diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig
new file mode 100644
index 0000000000..043a590353
--- /dev/null
+++ b/zig/src/families/lfm2_moe/dspark_test.zig
@@ -0,0 +1,272 @@
+//! The DSpark drafter's kernels on the GPU against host references: taps, the Markov step, half-split RoPE.
+const std = @import("std");
+const mtl = @import("metal");
+const c = @import("config.zig");
+const kernels = @import("kernels.zig");
+const quant = @import("dspark_quant.zig");
+const block = @import("dspark_block.zig");
+const Pick = @import("tree_round_gpu").contract.Pick;
+const ds = @import("dspark.zig");
+const lanes = @import("lanes");
+
+const shared = mtl.ResourceOptions.shared;
+
+pub const Rig = struct {
+ device: mtl.Device,
+ queue: mtl.Queue,
+ k: kernels.Kernels,
+
+ pub fn init() !Rig {
+ const device = mtl.Device.init() catch return error.SkipZigTest;
+ errdefer device.deinit();
+ const queue = try device.queue();
+ errdefer queue.deinit();
+ return .{ .device = device, .queue = queue, .k = try kernels.load(device) };
+ }
+
+ pub fn deinit(r: *Rig) void {
+ r.k.deinit();
+ r.queue.deinit();
+ r.device.deinit();
+ }
+
+ pub fn buffer(r: *Rig, comptime T: type, n: usize) !mtl.Buffer {
+ const b = try r.device.buffer(n * @sizeOf(T), shared);
+ @memset(b.slice(u8, n * @sizeOf(T)), 0);
+ return b;
+ }
+
+ fn run(r: *Rig, comptime key: []const u8, bufs: []const mtl.Buffer, value: anytype, at: usize, grid: [2]usize) !void {
+ const cb = r.queue.commandBuffer();
+ const e = cb.compute(.serial);
+ e.setPipeline(r.k.get(key));
+ for (bufs, 0..) |b, i| e.setBuffer(b, 0, if (i >= at) i + 1 else i);
+ e.setValue(value, at);
+ e.dispatchThreads(mtl.Size.of(grid[0], grid[1], 1), mtl.Size.of(@min(grid[0], 256), 1, 1));
+ e.end();
+ cb.commit();
+ cb.wait();
+ if (cb.failure()) |_| return error.GpuFailed;
+ }
+};
+
+pub fn noise(seed: u64, out: []u16, scale: f32) void {
+ var prng = std.Random.DefaultPrng.init(seed);
+ for (out) |*v| v.* = quant.toBf16((prng.random().float(f32) * 2 - 1) * scale);
+}
+
+test "lfm2 dspark: the tap kernel copies each row's hidden values into its tap's columns, the rest untouched" {
+ var r = try Rig.init();
+ defer r.deinit();
+ const rows = 5;
+ const taps = 3;
+ const h = try r.buffer(u16, rows * c.hidden);
+ defer h.deinit();
+ const out = try r.buffer(u16, rows * taps * c.hidden);
+ defer out.deinit();
+ noise(7, h.slice(u16, rows * c.hidden), 4);
+ try r.run("lfm2_tap", &.{ h, out }, [2]u32{ taps * c.hidden, c.hidden }, 2, .{ c.hidden / 8, rows });
+ const want = h.slice(u16, rows * c.hidden);
+ const got = out.slice(u16, rows * taps * c.hidden);
+ for (0..rows) |row| {
+ try std.testing.expectEqualSlices(u16, want[row * c.hidden ..][0..c.hidden], got[(row * taps + 1) * c.hidden ..][0..c.hidden]);
+ for (got[row * taps * c.hidden ..][0..c.hidden]) |v| try std.testing.expectEqual(@as(u16, 0), v);
+ for (got[(row * taps + 2) * c.hidden ..][0..c.hidden]) |v| try std.testing.expectEqual(@as(u16, 0), v);
+ }
+}
+
+test "lfm2 dspark: a Markov step picks the argmax of bf16(base + bf16(W2 . W1[prev])) over every threadgroup" {
+ var r = try Rig.init();
+ defer r.deinit();
+ const rank = 256;
+ const vocab = 1300; // three scan threadgroups, the last one short
+ const streams = 4;
+ const groups = (vocab + 511) / 512;
+ const w1 = try r.buffer(u16, vocab * rank);
+ defer w1.deinit();
+ noise(11, w1.slice(u16, vocab * rank), 1);
+ const codes = try r.buffer(u8, vocab * rank);
+ defer codes.deinit();
+ for (codes.slice(u8, vocab * rank), 0..) |*q, i| q.* = @truncate(i *% 2654435761 >> 7);
+ const sc = try r.buffer(u16, vocab * rank / 64);
+ defer sc.deinit();
+ const bi = try r.buffer(u16, vocab * rank / 64);
+ defer bi.deinit();
+ noise(12, sc.slice(u16, vocab * rank / 64), 0.01);
+ noise(13, bi.slice(u16, vocab * rank / 64), 0.5);
+ // a token every second word (the picks' layout): inside W1, its last row, past it (a zero row), and a NaN row
+ const ids = try r.buffer(u32, 2 * streams);
+ defer ids.deinit();
+ @memcpy(ids.slice(u32, 2 * streams), &[_]u32{ 17, 9, vocab - 1, 9, vocab + 5, 0, 3, 3 });
+ const logits = try r.buffer(u16, streams * 2 * vocab);
+ defer logits.deinit();
+ noise(14, logits.slice(u16, streams * 2 * vocab), 4);
+ logits.slice(u16, streams * 2 * vocab)[3 * 2 * vocab + 700] = 0x7fc0;
+ const best = try r.buffer(f32, streams * groups);
+ defer best.deinit();
+ const at = try r.buffer(u32, streams * groups);
+ defer at.deinit();
+ const picks = try r.buffer(u32, 2 * streams);
+ defer picks.deinit();
+ const cb = r.queue.commandBuffer();
+ const e = cb.compute(.serial);
+ e.setPipeline(r.k.get("lfm2_markov_scan"));
+ for ([_]mtl.Buffer{ logits, w1, codes, ids }, 0..) |b, i| e.setBuffer(b, 0, i);
+ e.setValue([4]u32{ 2 * vocab, rank, vocab, 2 }, 4);
+ for ([_]mtl.Buffer{ best, at, sc, bi }, 5..) |b, i| e.setBuffer(b, 0, i);
+ e.dispatchGroups(mtl.Size.of(groups, streams, 1), mtl.Size.of(256, 1, 1));
+ e.setPipeline(r.k.get("lfm2_markov_pick"));
+ e.setBuffer(best, 0, 0);
+ e.setBuffer(at, 0, 1);
+ e.setValue(@as(u32, groups), 2);
+ e.setBuffer(picks, 0, 3);
+ e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1));
+ e.end();
+ cb.commit();
+ cb.wait();
+ const got = picks.slice(u32, 2 * streams);
+ const base = logits.slice(u16, streams * 2 * vocab);
+ const id_list = ids.slice(u32, 2 * streams);
+ for (0..streams) |s| {
+ if (s == 3) { // the planted NaN: flagged
+ try std.testing.expectEqual(@as(u32, 1), got[2 * s + 1]);
+ continue;
+ }
+ try std.testing.expectEqual(@as(u32, 0), got[2 * s + 1]);
+ const id = id_list[2 * s];
+ var top: f64 = -std.math.inf(f64);
+ var mine: f64 = 0;
+ for (0..vocab) |v| {
+ var dot: f64 = 0;
+ for (0..rank) |i| {
+ const g = (v * rank + i) / 64;
+ const w = @as(f64, quant.toF32(sc.slice(u16, vocab * rank / 64)[g])) * @as(f64, @floatFromInt(codes.slice(u8, vocab * rank)[v * rank + i])) + quant.toF32(bi.slice(u16, vocab * rank / 64)[g]);
+ dot += w * (if (id < vocab) quant.toF32(w1.slice(u16, vocab * rank)[id * rank + i]) else 0);
+ }
+ const score: f64 = quant.toF32(quant.toBf16(quant.toF32(base[s * 2 * vocab + v]) + quant.toF32(quant.toBf16(@floatCast(dot)))));
+ top = @max(top, score);
+ if (v == got[2 * s]) mine = score;
+ }
+ // the pick is the best score, or within two bf16 steps of it where fp32 sums round apart
+ try std.testing.expect(got[2 * s] < vocab and mine >= top - @abs(top) * 0x1p-7);
+ }
+}
+
+test "lfm2 dspark: lfm2_qk on half-split rows and gains gives interleaved RoPE's keys, reordered" {
+ var r = try Rig.init();
+ defer r.deinit();
+ const hd = c.head_dim;
+ const pos: u32 = 777;
+ const theta: f32 = 1e7;
+ var k_il: [c.kv_heads * hd]u16 = undefined;
+ var gain_il: [hd]u16 = undefined;
+ noise(21, &k_il, 2);
+ noise(22, &gain_il, 1.5);
+ const q = try r.buffer(u16, c.query_heads * hd);
+ defer q.deinit();
+ const k = try r.buffer(u16, c.kv_heads * hd);
+ defer k.deinit();
+ const v = try r.buffer(u16, c.kv_heads * hd);
+ defer v.deinit();
+ const qn = try r.buffer(u16, hd);
+ defer qn.deinit();
+ const kn = try r.buffer(u16, hd);
+ defer kn.deinit();
+ for (k.slice(u16, c.kv_heads * hd), 0..) |*x, i| x.* = k_il[quant.halfSplit(i, hd)];
+ for (kn.slice(u16, hd), 0..) |*x, i| x.* = gain_il[quant.halfSplit(i, hd)];
+ @memset(qn.slice(u16, hd), quant.toBf16(1));
+ const cap = 1024;
+ const queries = try r.buffer(u16, c.query_heads * hd);
+ defer queries.deinit();
+ const keys = try r.buffer(u16, c.kv_heads * cap * hd);
+ defer keys.deinit();
+ const values = try r.buffer(u16, c.kv_heads * cap * hd);
+ defer values.deinit();
+ const Args = extern struct { len: u32, cap: u32, eps: f32, log2_theta: f32 };
+ const cb = r.queue.commandBuffer();
+ const e = cb.compute(.serial);
+ e.setPipeline(r.k.get("lfm2_qk"));
+ for ([_]mtl.Buffer{ q, k, v, qn, kn }, 0..) |b, i| e.setBuffer(b, 0, i);
+ const args = Args{ .len = pos, .cap = cap, .eps = c.eps, .log2_theta = std.math.log2(theta) };
+ e.setBytes(std.mem.asBytes(&args), 5);
+ for ([_]mtl.Buffer{ queries, keys, values }, 6..) |b, i| e.setBuffer(b, 0, i);
+ e.dispatchGroups(mtl.Size.of(c.query_heads + c.kv_heads, 1, 1), mtl.Size.of(32, 1, 1));
+ e.end();
+ cb.commit();
+ cb.wait();
+ const got = keys.slice(u16, c.kv_heads * cap * hd);
+ for (0..c.kv_heads) |h| {
+ var ss: f32 = 0;
+ for (0..hd) |i| ss += quant.toF32(k_il[h * hd + i]) * quant.toF32(k_il[h * hd + i]);
+ const inv = 1 / @sqrt(ss / hd + c.eps);
+ for (0..hd / 2) |i| {
+ const angle = @as(f32, pos) * std.math.pow(f32, theta, -@as(f32, @floatFromInt(i)) / (hd / 2));
+ const a = quant.toF32(k_il[h * hd + 2 * i]) * inv * quant.toF32(gain_il[2 * i]);
+ const b = quant.toF32(k_il[h * hd + 2 * i + 1]) * inv * quant.toF32(gain_il[2 * i + 1]);
+ const want = [2]f32{ a * @cos(angle) - b * @sin(angle), a * @sin(angle) + b * @cos(angle) };
+ const at = (h * cap + pos) * hd;
+ const have = [2]f32{ quant.toF32(got[at + i]), quant.toF32(got[at + i + hd / 2]) };
+ for (want, have) |w, x| try std.testing.expect(@abs(w - x) <= 0.02 * @max(1, @abs(w)));
+ }
+ }
+}
+test "lfm2 dspark: a settled hold moves the timed passes at its block counts a tenth of the way to its GPU ms" {
+ var d: ds.Drafter = undefined;
+ const grid = [_]lanes.hold_price.HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1 }, .{ .streams = 4, .depth = 1, .ms = 2 }, .{ .streams = 4, .depth = 4, .ms = 4 }, .{ .streams = 8, .depth = 1, .ms = 3 } };
+ @memcpy(d.holds[0..grid.len], &grid);
+ d.holds_len = grid.len;
+ d.pass_len = 4;
+ d.depths[0..4].* = .{ 1, 1, 1, 1 };
+ d.priceHold(4); // priced 2 ms: the 4-block passes rise by a tenth of the doubling
+ try std.testing.expectApproxEqAbs(@as(f64, 2.2), d.holds[1].ms, 1e-12);
+ try std.testing.expectApproxEqAbs(@as(f64, 4.4), d.holds[2].ms, 1e-12);
+ try std.testing.expectEqual(@as(f64, 1), d.holds[0].ms);
+ try std.testing.expectEqual(@as(f64, 3), d.holds[3].ms);
+ d.pass_len = 6; // between 4 and 8 blocks: both move
+ d.depths[0..6].* = .{ 1, 1, 1, 1, 1, 1 };
+ d.priceHold(0);
+ try std.testing.expectEqual(@as(f64, 2.2), d.holds[1].ms); // no GPU time: nothing learned
+ d.priceHold(2.6 * 2);
+ try std.testing.expect(d.holds[1].ms > 2.2 and d.holds[3].ms > 3 and d.holds[0].ms == 1);
+}
+
+/// lfm2_draft_ids on one pick (token, nonfinite flag) at a stride of 4 words: the id it wrote and the bad flag.
+fn draftIds(r: *Rig, pick: Pick) ![2]u32 {
+ const picks = try r.buffer(u32, 4);
+ defer picks.deinit();
+ const ids = try r.buffer(u32, 1);
+ defer ids.deinit();
+ const bad = try r.buffer(u32, 1);
+ defer bad.deinit();
+ picks.slice(u32, 4)[0] = pick.token;
+ picks.slice(u32, 4)[1] = pick.nonfinite;
+ try r.run("lfm2_draft_ids", &.{ picks, ids, bad }, [4]u32{ 4, 1, c.vocab, 0 }, 1, .{ 1, 1 });
+ return .{ ids.slice(u32, 1)[0], bad.slice(u32, 1)[0] };
+}
+
+test "lfm2 dspark: held ids carry a flagged or invalid pick as token 0 and the bad flag, a good pick untouched" {
+ var r = try Rig.init();
+ defer r.deinit();
+ try std.testing.expectEqual([2]u32{ 5, 0 }, try draftIds(&r, .{ .token = 5 }));
+ try std.testing.expectEqual([2]u32{ c.vocab - 1, 0 }, try draftIds(&r, .{ .token = c.vocab - 1 }));
+ for ([_]u32{ 1, 2, 4, 7 }) |kind| try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = 9, .nonfinite = kind }));
+ try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = std.math.maxInt(u32) }));
+ try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = c.vocab }));
+}
+
+test "lfm2 dspark: a settled hold refuses a NaN, infinity or invalid pick as the synchronous path does" {
+ var picks: [4 * ds.pass_rows]Pick = @splat(.{ .token = 3 });
+ var a: [4]u32 = undefined;
+ var b: [4]u32 = undefined;
+ const chains = [_][]u32{ &a, &b };
+ try block.copyPicks(&picks, &chains, &.{ 2, 2 });
+ try std.testing.expectEqualSlices(u32, &.{ 3, 3 }, a[0..2]);
+ const bad = [_]Pick{ .{ .token = 3, .nonfinite = 1 }, .{ .token = 3, .nonfinite = 2 }, .{ .token = 3, .nonfinite = 4 }, .{ .token = std.math.maxInt(u32) }, .{ .token = c.vocab } };
+ for (bad) |p| {
+ picks[ds.pass_rows + 1] = p; // the second block's second step
+ try std.testing.expectError(error.DraftNonfinite, block.copyPicks(&picks, &chains, &.{ 2, 2 }));
+ picks[ds.pass_rows + 1] = .{ .token = 3 };
+ }
+ picks[3 * ds.pass_rows] = .{ .nonfinite = 1 }; // a step past the block's depth is not read
+ try block.copyPicks(&picks, &chains, &.{ 2, 2 });
+}
diff --git a/zig/src/families/lfm2_moe/dspark_weights.zig b/zig/src/families/lfm2_moe/dspark_weights.zig
new file mode 100644
index 0000000000..9d87dd6eed
--- /dev/null
+++ b/zig/src/families/lfm2_moe/dspark_weights.zig
@@ -0,0 +1,178 @@
+//! The DSpark drafter's weights: the released bf16 linears as stored, or on request quantized to 8-bit affine codes.
+const std = @import("std");
+const mtl = @import("metal");
+const c = @import("config.zig");
+const dc = @import("dspark_config.zig");
+const quant = @import("dspark_quant.zig");
+const ckpt = @import("../../core/checkpoint_metal.zig");
+const shards = @import("../../core/checkpoint.zig");
+const tile = @import("../../core/tile_qmm.zig");
+
+const shared = mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked;
+
+/// An [n, k] bf16 matrix as released (rows only reordered for half-split RoPE where the checkpoint is read).
+pub const Bf16 = struct { w: mtl.Buffer, n: usize, k: usize };
+
+/// A linear: bf16 as stored (bits 0), or MLX affine codes at the loader's width.
+pub const Linear = union(enum) { bf16: Bf16, codes: tile.Matrix };
+
+/// A bf16 vector in its own buffer (norm gains; RoPE-paired ones in the half-split order the kernels rotate).
+pub const Vector = mtl.Buffer;
+
+pub const Layer = struct {
+ q: Linear,
+ k: Linear,
+ v: Linear,
+ o: Linear,
+ gate: Linear,
+ up: Linear,
+ down: Linear,
+ input_norm: Vector,
+ post_norm: Vector,
+ q_norm: Vector,
+ k_norm: Vector,
+};
+
+pub const Weights = struct {
+ fc: Linear,
+ hidden_norm: Vector,
+ norm: Vector,
+ layers: [8]Layer,
+ /// The Markov head: W1 [vocab, rank] bf16 as stored (rows gathered by token), W2 [vocab, rank] as the linears.
+ w1: mtl.Buffer,
+ w2: Linear,
+ /// The confidence head: its [hidden + rank] bf16 weights and fp32 bias (null: the checkpoint has none).
+ confidence: ?mtl.Buffer = null,
+ confidence_bias: f32 = 0,
+ owned: std.ArrayList(mtl.Buffer) = .empty,
+
+ pub fn deinit(w: *Weights, gpa: std.mem.Allocator) void {
+ for (w.owned.items) |b| b.deinit();
+ w.owned.deinit(gpa);
+ }
+
+ /// Every buffer the weights hold, for a residency set.
+ pub fn buffers(w: *const Weights) []const mtl.Buffer {
+ return w.owned.items;
+ }
+};
+
+const Loader = struct {
+ gpa: std.mem.Allocator,
+ device: mtl.Device,
+ check: *const ckpt.Checkpoint,
+ stage: *ckpt.Stage, // a matrix read from the file, one at a time, into one reused buffer
+ owned: *std.ArrayList(mtl.Buffer),
+ bits: u8,
+
+ fn buffer(l: Loader, bytes: usize) !mtl.Buffer {
+ const b = try l.device.buffer(bytes, shared);
+ errdefer b.deinit();
+ try l.owned.append(l.gpa, b);
+ return b;
+ }
+
+ /// A vector as loaded, or a matrix read into the stage (valid until the next matrix), of `shape` in bf16.
+ fn tensor(l: Loader, name: []const u8, shape: []const usize) !ckpt.Tensor {
+ l.stage.reset();
+ if (shape.len == 2) try l.stage.reserve(l.check, &.{name});
+ const t = if (shape.len == 2) try l.stage.read(l.check, name) else try l.check.get(name);
+ if (t.dtype != .bf16 or t.rank != shape.len or !std.mem.eql(usize, t.shape[0..t.rank], shape)) {
+ std.log.err("{s}: unexpected DSpark tensor dtype or shape", .{name});
+ return error.BadDraftWeight;
+ }
+ return t;
+ }
+
+ /// A bf16 vector copied out, its pairs reordered for half-split RoPE when `paired`.
+ fn vector(l: Loader, name: []const u8, len: usize, paired: bool) !Vector {
+ const src = (try l.tensor(name, &.{len})).host(u16);
+ const b = try l.buffer(len * 2);
+ const dst = b.slice(u16, len);
+ for (dst, 0..) |*v, i| v.* = src[if (paired) quant.halfSplit(i, len) else i];
+ return b;
+ }
+
+ /// An [n, k] bf16 matrix as it is or as affine codes; rows reordered for half-split RoPE when `paired`.
+ fn matrix(l: Loader, name: []const u8, n: usize, k: usize, paired: bool) !Linear {
+ const t = try l.tensor(name, &.{ n, k });
+ var rows = t.host(u16);
+ var moved: ?[]u16 = null;
+ defer if (moved) |m| l.gpa.free(m);
+ if (paired) {
+ const m = try l.gpa.alloc(u16, n * k);
+ moved = m;
+ for (0..n) |r| @memcpy(m[r * k ..][0..k], rows[quant.halfSplit(r, c.head_dim) * k ..][0..k]);
+ rows = m;
+ }
+ if (l.bits == 0) {
+ const b = try l.buffer(n * k * 2);
+ @memcpy(b.slice(u16, n * k), rows);
+ return .{ .bf16 = .{ .w = b, .n = n, .k = k } };
+ }
+ const codes = try l.buffer(n * k * l.bits / 8);
+ const scales = try l.buffer(n * k / 64 * 2);
+ const biases = try l.buffer(n * k / 64 * 2);
+ const job: quant.Job = .{ .src = rows, .k = k, .bits = l.bits, .codes = codes.slice(u8, n * k * l.bits / 8), .scales = scales.slice(u16, n * k / 64), .biases = biases.slice(u16, n * k / 64) };
+ try job.all(l.gpa, n);
+ return .{ .codes = .{ .w = codes, .scales = scales, .biases = biases, .n = n, .k = k, .bits = l.bits } };
+ }
+};
+
+fn part(buf: []u8, layer: usize, name: []const u8) ![]const u8 {
+ return std.fmt.bufPrint(buf, "layers.{d}.{s}.weight", .{ layer, name });
+}
+
+/// Matrices stay in the file until the loader reads them; vectors load with the header.
+fn matrices(_: []const u8, rank: usize) bool {
+ return rank == 2;
+}
+
+/// The checkpoint read as released (`bits` 0) or quantized to 8-bit codes, a matrix at a time; buffers freed after.
+pub fn load(gpa: std.mem.Allocator, device: mtl.Device, io: std.Io, dir: []const u8, cfg: *const dc.Config, bits: u8) !Weights {
+ if (bits != 0 and bits != 8) return error.UnsupportedDraftBits;
+ var check = ckpt.Checkpoint.init(gpa);
+ defer check.deinit();
+ var stage: ckpt.Stage = .{ .device = device };
+ defer stage.deinit();
+ const files = try shards.shardFiles(gpa, io, dir);
+ defer shards.freeShardFiles(gpa, files);
+ for (files) |path| try check.addFileSplit(device, path, "", null, matrices);
+ var w: Weights = undefined;
+ w.owned = .empty;
+ errdefer w.deinit(gpa);
+ const l: Loader = .{ .gpa = gpa, .device = device, .check = &check, .stage = &stage, .owned = &w.owned, .bits = bits };
+ const qw = c.query_heads * c.head_dim;
+ const kw = c.kv_heads * c.head_dim;
+ w.fc = try l.matrix("fc.weight", c.hidden, cfg.tap_count * c.hidden, false);
+ w.hidden_norm = try l.vector("hidden_norm.weight", c.hidden, false);
+ w.norm = try l.vector("norm.weight", c.hidden, false);
+ var name: [128]u8 = undefined;
+ for (w.layers[0..cfg.layers], 0..) |*ly, i| {
+ ly.q = try l.matrix(try part(&name, i, "self_attn.q_proj"), qw, c.hidden, true);
+ ly.k = try l.matrix(try part(&name, i, "self_attn.k_proj"), kw, c.hidden, true);
+ ly.v = try l.matrix(try part(&name, i, "self_attn.v_proj"), kw, c.hidden, false);
+ ly.o = try l.matrix(try part(&name, i, "self_attn.o_proj"), c.hidden, qw, false);
+ ly.gate = try l.matrix(try part(&name, i, "mlp.gate_proj"), cfg.intermediate, c.hidden, false);
+ ly.up = try l.matrix(try part(&name, i, "mlp.up_proj"), cfg.intermediate, c.hidden, false);
+ ly.down = try l.matrix(try part(&name, i, "mlp.down_proj"), c.hidden, cfg.intermediate, false);
+ ly.input_norm = try l.vector(try part(&name, i, "input_layernorm"), c.hidden, false);
+ ly.post_norm = try l.vector(try part(&name, i, "post_attention_layernorm"), c.hidden, false);
+ ly.q_norm = try l.vector(try part(&name, i, "self_attn.q_norm"), c.head_dim, true);
+ ly.k_norm = try l.vector(try part(&name, i, "self_attn.k_norm"), c.head_dim, true);
+ }
+ const w1 = (try l.tensor("markov_head.markov_w1.weight", &.{ c.vocab, cfg.rank })).host(u16);
+ w.w1 = try l.buffer(c.vocab * cfg.rank * 2);
+ @memcpy(w.w1.slice(u16, c.vocab * cfg.rank), w1);
+ w.w2 = try l.matrix("markov_head.markov_w2.weight", c.vocab, cfg.rank, false);
+ w.confidence = null;
+ w.confidence_bias = 0;
+ if (check.has("confidence_head.proj.weight")) {
+ const width = c.hidden + cfg.rank; // the final norm's row, then W1 of the token before the draft
+ const proj = (try l.tensor("confidence_head.proj.weight", &.{ 1, width })).host(u16);
+ w.confidence = try l.buffer(width * 2);
+ @memcpy(w.confidence.?.slice(u16, width), proj);
+ w.confidence_bias = quant.toF32((try l.tensor("confidence_head.proj.bias", &.{1})).host(u16)[0]);
+ }
+ return w;
+}
diff --git a/zig/src/families/lfm2_moe/forward.zig b/zig/src/families/lfm2_moe/forward.zig
index 212b5978bc..51408e2fd4 100644
--- a/zig/src/families/lfm2_moe/forward.zig
+++ b/zig/src/families/lfm2_moe/forward.zig
@@ -118,7 +118,17 @@ pub fn matrix(w: row.Weights) tile.Matrix {
return .{ .w = w.w, .w_off = w.w_off, .scales = w.scales, .s_off = w.s_off, .biases = w.biases, .b_off = w.b_off, .n = w.n, .k = w.k, .bits = w.bits };
}
+/// A window's held drafts on the device: `count` tokens at `src` + `off`, `stride` words apart, into ids from `row`.
+pub const Held = struct { row: usize, src: Buffer, off: usize, stride: u32, count: u32 };
+/// A GPU event the forward waits on before it reads held drafts, and the value it waits for.
+pub const Wait = struct { event: mtl.Event, value: u64 };
+
pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const u32, record: bool, head: Head) !void {
+ return runHeld(m, s, segments, tokens, record, head, &.{}, null);
+}
+
+/// `run` with held drafts copied into the rows' ids on the GPU first (after `wait`): their host ids are ignored.
+pub fn runHeld(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const u32, record: bool, head: Head, held: []const Held, wait: ?Wait) !void {
if (tokens.len == 0 or tokens.len > s.rows or (record and tokens.len > st.batch_rows)) return error.LfmRowsExceeded;
var total: usize = 0;
for (segments) |seg| {
@@ -131,10 +141,20 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const
const pool = mtl.objc.Pool.push();
defer pool.pop();
const cb = m.queue.commandBuffer();
+ if (wait) |w| cb.waitForEvent(w.event, w.value);
const prompt = (!record and head == .last) or total >= tile_rows;
const e = cb.compute(.serial);
const plan = if (segs.batched(record, segments.len)) segs.prepare(e, s, segments) else null;
const enc = Encoder{ .e = e, .m = m, .s = s, .prompt = prompt, .record = record, .plan = plan };
+ if (held.len > 0) @memset(s.bad.contents()[0..4], 0);
+ for (held) |h| {
+ enc.pipe("lfm2_draft_ids");
+ e.setBuffer(h.src, h.off, 0);
+ e.setValue([4]u32{ h.stride, h.count, c.vocab, 0 }, 1);
+ e.setBuffer(s.ids, h.row * 4, 2);
+ e.setBuffer(s.bad, 0, 3);
+ enc.threads(.{ h.count, 1, 1 }, .{ 32, 1, 1 });
+ }
const embedding = m.weights.embedding;
e.setPipeline(m.kernels.embed(embedding.bits));
e.setBuffer(s.ids, 0, 0);
@@ -146,6 +166,7 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const
for (m.weights.blocks, 0..) |block, i| {
enc.norm(s, block.operator_norm, i == 0, total, combine);
combine = false;
+ if (i > 0) tap(enc, s, i - 1, total);
switch (block.mixer) {
.conv => |cv| conv(enc, s, cv, i, segments, total),
.attention => |a| attention(enc, s, a, &m.config, segments, i, total),
@@ -162,6 +183,7 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const
}
}
enc.norm(s, m.weights.norm, false, total, combine);
+ tap(enc, s, c.layers - 1, total);
if (head == .all) {
enc.projection(embedding, s.x, 0, s.logits, 0, total);
enc.pipe("lfm2_argmax");
@@ -178,6 +200,7 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const
std.debug.print("LFM2 forward: {s}\n", .{msg});
return error.GpuFailed;
}
+ if (held.len > 0 and std.mem.readInt(u32, s.bad.contents()[0..4], .little) != 0) return error.DraftNonfinite;
var base: usize = 0;
for (segments) |seg| {
const logit_row = if (head == .all) base + seg.rows - 1 else 0;
@@ -187,6 +210,18 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const
}
}
+/// Layer `layer`'s output rows (h, just summed by the next norm) into each tap of it, when a drafter reads them.
+fn tap(enc: Encoder, s: *st.Scratch, layer: usize, total: usize) void {
+ const taps = s.taps orelse return;
+ for (taps.layers[0..taps.count], 0..) |l, t| if (l == layer) {
+ enc.pipe("lfm2_tap");
+ enc.e.setBuffer(s.h, 0, 0);
+ enc.e.setBuffer(taps.buf, 0, 1);
+ enc.e.setValue([2]u32{ taps.count * c.hidden, @intCast(t * c.hidden) }, 2);
+ enc.threads(.{ c.hidden / 8, total, 1 }, .{ 256, 1, 1 });
+ };
+}
+
fn conv(enc: Encoder, s: *st.Scratch, cv: wts.Conv, layer: usize, segments: []const Segment, total: usize) void {
const e = enc.e;
enc.projection(cv.in, s.x, 0, s.bcx, 0, total);
diff --git a/zig/src/families/lfm2_moe/kernels.zig b/zig/src/families/lfm2_moe/kernels.zig
index aa1e798fc0..c83c9a0b4a 100644
--- a/zig/src/families/lfm2_moe/kernels.zig
+++ b/zig/src/families/lfm2_moe/kernels.zig
@@ -16,6 +16,8 @@ const names = [_][:0]const u8{
"lfm2_attn_mma",
"lfm2_attn_mma_r2",
"lfm2_attn_merge",
+ "lfm2_attn_block",
+ "lfm2_attn_block_merge",
"lfm2_route_logits",
"lfm2_route_select",
"lfm2_swiglu",
@@ -27,6 +29,14 @@ const names = [_][:0]const u8{
"lfm2_qk_segs",
"lfm2_attn_segs",
"lfm2_attn_merge_segs",
+ "lfm2_tap",
+ "lfm2_markov_scan",
+ "lfm2_markov_scan_bf16",
+ "lfm2_bf16_rows",
+ "lfm2_bf16_mma",
+ "lfm2_markov_pick",
+ "lfm2_draft_ids",
+ "lfm2_dspark_confidence",
};
pub const Kernels = struct {
diff --git a/zig/src/families/lfm2_moe/lfm2_moe.zig b/zig/src/families/lfm2_moe/lfm2_moe.zig
index 97f9cbd5d8..8f861e31b4 100644
--- a/zig/src/families/lfm2_moe/lfm2_moe.zig
+++ b/zig/src/families/lfm2_moe/lfm2_moe.zig
@@ -7,3 +7,7 @@ pub const state = @import("state.zig");
pub const forward = @import("forward.zig");
pub const backend = @import("backend.zig");
pub const snapshot = @import("snapshot.zig");
+pub const dspark = @import("dspark.zig");
+pub const dspark_config = @import("dspark_config.zig");
+pub const dspark_block = @import("dspark_block.zig");
+pub const dspark_quant = @import("dspark_quant.zig");
diff --git a/zig/src/families/lfm2_moe/snapshot.zig b/zig/src/families/lfm2_moe/snapshot.zig
index d1c1a8b3ae..d7289331b7 100644
--- a/zig/src/families/lfm2_moe/snapshot.zig
+++ b/zig/src/families/lfm2_moe/snapshot.zig
@@ -4,7 +4,7 @@ const c = @import("config.zig");
const st = @import("state.zig");
/// One state: conv states by conv layer, then keys and values [kv head][at][head dim] by attention layer.
-pub const Snap = struct { at: u32, bytes: []u8 };
+pub const Snap = struct { at: u32, bytes: []u8, draft: ?[]u8 = null };
/// A row's keys or values for every kv head of one attention layer.
const row_bytes = c.kv_heads * c.head_dim * 2;
@@ -56,6 +56,7 @@ pub fn restore(cache: *st.Cache, snap: *const Snap) !void {
}
pub fn drop(gpa: std.mem.Allocator, snap: *Snap) void {
+ if (snap.draft) |d| gpa.free(d); // the drafter's context at `at`, when one was kept with the state
gpa.free(snap.bytes);
gpa.destroy(snap);
}
diff --git a/zig/src/families/lfm2_moe/state.zig b/zig/src/families/lfm2_moe/state.zig
index 3242d8bf26..883f670fbe 100644
--- a/zig/src/families/lfm2_moe/state.zig
+++ b/zig/src/families/lfm2_moe/state.zig
@@ -51,6 +51,12 @@ pub const Storage = struct {
};
pub const Attention = struct { keys: mtl.Buffer, values: mtl.Buffer };
+/// Layers a drafter reads at most (each layer's output, hidden wide).
+pub const max_taps = 8;
+/// The drafter's layers in its order and every forward row's outputs of them, [rows][count][hidden] bf16.
+pub const Taps = struct { layers: [max_taps]u32 = @splat(0), count: u32 = 0, buf: mtl.Buffer };
+/// Where a stream's latest tapped rows sit in the scratch: cache rows [start, start + rows) from scratch row `row`.
+pub const TapRows = struct { start: usize, row: usize, rows: usize, gen: u64 };
pub const Mixer = union(enum) { conv: mtl.Buffer, attention: Attention };
pub const Cache = struct {
@@ -60,6 +66,9 @@ pub const Cache = struct {
last: ?struct { start: usize, base: usize, rows: usize } = null,
blocks: [c.layers]Mixer,
logits: mtl.Buffer,
+ /// The stream's latest tapped rows in the scratch, and its prompt's last row's taps (kept for the first draft).
+ tap: ?TapRows = null,
+ last_tap: ?mtl.Buffer = null,
pub fn init(gpa: std.mem.Allocator, device: mtl.Device, attention: *const [c.layers]bool, capacity: usize) !Cache {
var out: Cache = .{ .memory = .{ .gpa = gpa, .device = device }, .capacity = capacity, .blocks = undefined, .logits = undefined };
@@ -110,6 +119,7 @@ pub const Scratch = struct {
memory: Storage,
rows: usize,
ids: mtl.Buffer,
+ bad: mtl.Buffer, // set by lfm2_draft_ids when a held pick is flagged nonfinite or past the vocabulary
h: mtl.Buffer,
x: mtl.Buffer,
r: mtl.Buffer,
@@ -148,6 +158,8 @@ pub const Scratch = struct {
segs: mtl.Buffer,
rowseg: mtl.Buffer,
tiles: mtl.Buffer,
+ /// A drafter's taps of every row, once the backend is told its layers (null: none kept).
+ taps: ?Taps = null,
pub fn init(gpa: std.mem.Allocator, device: mtl.Device, attention: *const [c.layers]bool, rows: usize) !Scratch {
var out: Scratch = undefined;
@@ -155,9 +167,11 @@ pub const Scratch = struct {
errdefer out.memory.deinit();
out.rows = rows;
out.snapshots = @splat(null);
+ out.taps = null;
const mem = &out.memory;
const slots = rows * c.topk;
out.ids = try mem.alloc(rows * 4);
+ out.bad = try mem.alloc(4);
inline for (.{ "h", "x", "r", "y", "q" }) |name| @field(out, name) = try mem.alloc(rows * c.hidden * 2);
out.queries = try mem.alloc((rows + attn_rows) * c.hidden * 2); // a prompt attention tile reads attn_rows from its first
out.bcx = try mem.alloc(rows * 3 * c.hidden * 2);
diff --git a/zig/src/native/cuda_options.zig b/zig/src/native/cuda_options.zig
index d6a6eefe28..0c07152bdb 100644
--- a/zig/src/native/cuda_options.zig
+++ b/zig/src/native/cuda_options.zig
@@ -36,6 +36,10 @@ pub fn set(comptime F: type, o: api.Open, opts: *F.Options, problem: *[]const u8
return false;
};
}
+ if (o.drafter_bits_given and o.drafter_bits == 8) {
+ problem.* = "--drafter-bits 8: only LFM2.5-24B-A2B's Metal DSpark drafter takes it; this family takes 0 (bf16) or 4";
+ return false;
+ }
if (comptime @hasField(F.Options, "drafter")) opts.drafter = o.drafter;
if (comptime @hasField(F.Options, "drafter_bits")) opts.drafter_bits = o.drafter_bits;
if (@hasField(F.Options, "kv8")) opts.kv8 = o.kv8;
diff --git a/zig/src/native/draft_sources_metal_test.zig b/zig/src/native/draft_sources_metal_test.zig
index 2bd5851477..0c295d80f4 100644
--- a/zig/src/native/draft_sources_metal_test.zig
+++ b/zig/src/native/draft_sources_metal_test.zig
@@ -20,6 +20,7 @@ test "draft sources: Metal Qwen2B and LFM2 MoE follow qualified verification wid
q.options.streams = 2;
l.options.streams = 2;
l.timed = 0;
+ l.shared_timed = 0;
for ([_]u32{ 1, 16 }) |width| {
q.width = width;
l.width = width;
diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig
index 5340d807de..3734c41a3c 100644
--- a/zig/src/native/lfm2_moe.zig
+++ b/zig/src/native/lfm2_moe.zig
@@ -1,4 +1,4 @@
-//! LFM2-MoE served by the existing lane host and scheduler.
+//! LFM2-MoE served by the existing lane host and scheduler, with LiquidAI's DSpark drafter from --drafter.
const std = @import("std");
const mtl = @import("metal");
const api = @import("engine_api");
@@ -8,11 +8,16 @@ const lanes = tf.lanes;
const cache_fit = @import("cache_fit.zig");
const metal_admit = @import("metal_admit.zig");
+/// --drafter loads LiquidAI's DSpark drafter (LiquidAI/LFM2.5-24B-A2B-DSpark) in front of the engine.
+pub const takes_drafter = true;
+
const Host = struct {
resident: ?mtl.resident.Resident = null,
gpa: std.mem.Allocator,
model: *q.Model,
metal: *q.backend.Metal,
+ draft: ?*q.dspark.Drafter = null,
+ wrap: ?lanes.drafted.Drafted = null,
warm: mtl.keepalive.Target = undefined, // the model's queue, for the lane host's idle ticker
config: lanes.Config,
clock: lanes.backend.WallClock,
@@ -25,7 +30,9 @@ const Host = struct {
h.host.stop();
if (h.cache) |*store| store.deinit();
h.core.deinit();
+ if (h.wrap) |*w| w.deinit();
mtl.resident.streams.drop(h);
+ if (h.draft) |d| d.deinit();
h.config.deinit(h.gpa);
h.metal.deinit();
h.model.deinit();
@@ -35,6 +42,13 @@ const Host = struct {
const cannot = "the native LFM2 engine cannot load this checkpoint ({s})";
+/// A refused open's backend, model and host, freed before it reports why.
+fn release(h: *Host) void {
+ h.metal.deinit();
+ h.model.deinit();
+ h.gpa.destroy(h);
+}
+
pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Open, problem: *[]const u8) !?api.Opened {
const pool = mtl.objc.Pool.push();
defer pool.pop();
@@ -65,18 +79,49 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope
const chunk = 2048;
h.metal = try q.backend.Metal.init(gpa, model, .{ .capacity = window, .chunk = chunk, .streams = @max(o.lanes, 2) });
errdefer h.metal.deinit();
- const per_stream = q.state.cacheBytes(&model.config.attention, window);
+ h.draft = null;
+ h.wrap = null;
+ if (if (o.drafts) o.drafter else null) |dir| {
+ if (h.metal.width == 1) {
+ problem.* = "--drafter: LFM2 wider lanes failed the local exactness check; drafts require verified wider lanes";
+ release(h);
+ return null;
+ }
+ const bits: u8 = if (o.drafter_bits_given) o.drafter_bits else 0;
+ if (bits != 0 and bits != 8) {
+ problem.* = "--drafter-bits: the LFM2 DSpark drafter takes 0 (bf16 as released, the default) or 8 (8-bit codes quantized at load)";
+ release(h);
+ return null;
+ }
+ h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window, .bits = bits }) catch |err| {
+ problem.* = try std.fmt.allocPrint(a, "the native LFM2 drafter cannot load {s} ({s}): it takes LiquidAI/LFM2.5-24B-A2B-DSpark", .{ dir, @errorName(err) });
+ release(h);
+ return null;
+ };
+ }
+ errdefer if (h.draft) |d| d.deinit();
+ // a stream's caches, and its drafter lane when one drafts
+ const per_stream = q.state.cacheBytes(&model.config.attention, window) + if (h.draft) |d| d.laneBytes(window) else 0;
const served = try metal_admit.streams(a, "lfm2", per_stream, metal_admit.room(model.device), o, window, problem) orelse {
- h.metal.deinit();
- model.deinit();
- gpa.destroy(h);
+ if (h.draft) |d| d.deinit();
+ release(h);
return null;
};
h.metal.options.streams = @max(served, 2);
- h.config = try lanes.Config.init(gpa, h.metal.facts(), q.state.window_rows, q.state.window_rows - 1);
+ if (h.draft) |d| {
+ h.wrap = try lanes.drafted.Drafted.init(gpa, h.metal.backend(), d.drafter());
+ // the verify copies the drafts on the GPU after the unwaited hold
+ h.metal.held_source = d.heldSource();
+ h.wrap.?.held_on_target = true;
+ }
+ errdefer if (h.wrap) |*w| w.deinit();
+ var facts = if (h.wrap) |*w| w.facts(h.metal.facts()) else h.metal.facts();
+ // a drafting stream prices its copy windows by its own copies: on distinct prompts they keep breaking
+ if (h.wrap != null) facts.own_copy_rates = true;
+ h.config = try lanes.Config.init(gpa, facts, q.state.window_rows, q.state.window_rows - 1);
errdefer h.config.deinit(gpa);
h.clock = .{ .io = io };
- h.core = lanes.Engine.init(gpa, &h.config, h.metal.backend(), h.clock.clock());
+ h.core = lanes.Engine.init(gpa, &h.config, if (h.wrap) |*w| w.backend() else h.metal.backend(), h.clock.clock());
errdefer h.core.deinit();
h.host = api.LaneHost.init(gpa, io, &h.core, .{ .lanes = served, .context_window = @intCast(window), .prefill_step = chunk, .structures = true });
h.cache = null;
@@ -97,8 +142,8 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope
if (plan.budget_bytes > 0) {
const S = q.backend.Snaps;
h.cache = api.prompt_cache.Store.init(gpa, .{
- .ptr = h.metal,
- .vtable = &.{ .bytes = S.bytes, .save = S.save, .restore = S.restore, .drop = S.drop },
+ .ptr = h,
+ .vtable = &.{ .bytes = Snaps.bytes, .save = Snaps.save, .restore = S.restore, .drop = Snaps.drop },
}, .{ .chunk = chunk }, plan.budget_bytes);
h.host.cache = &h.cache.?;
h.host.info_.prompt_cache = true;
@@ -106,7 +151,40 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope
errdefer if (h.cache) |*store| store.deinit();
try mtl.resident.streams.hold(h, io, "lfm2 moe");
errdefer mtl.resident.streams.drop(h);
+ if (h.draft) |d| if (h.resident) |*r| {
+ const held = try d.buffers(gpa);
+ defer gpa.free(held);
+ r.add(held);
+ };
h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick };
try h.host.start();
+ if (h.draft) |d| {
+ const mode = if (d.options.bits == 8) "8-bit codes quantized at load (--drafter-bits 8), not the released bf16" else "bf16 matrices as released";
+ std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, {s}", .{ d.config.block, mode });
+ }
return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h };
}
+
+/// The prompt cache's copies of a stream: the target's state at a mark (backend.zig), and the drafter's context there.
+const Snaps = struct {
+ const S = q.backend.Snaps;
+ fn host(ptr: *anyopaque) *Host {
+ return @ptrCast(@alignCast(ptr));
+ }
+ fn bytes(ptr: *anyopaque, at: u32) u64 {
+ const h = host(ptr);
+ return S.bytes(h.metal, at) + if (h.draft) |d| d.keptBytes(at) else 0;
+ }
+ fn save(ptr: *anyopaque, owner: ?*anyopaque, at: u32) anyerror!*anyopaque {
+ const h = host(ptr);
+ const saved = try S.save(h.metal, owner, at);
+ if (h.draft) |d| d.keepLane(@ptrCast(@alignCast(owner.?)), @ptrCast(@alignCast(saved))) catch |err| {
+ S.drop(h.metal, saved);
+ return err;
+ };
+ return saved;
+ }
+ fn drop(ptr: *anyopaque, saved: *anyopaque) void {
+ S.drop(host(ptr).metal, saved);
+ }
+};
diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig
index 1391c64f86..fbad2ae996 100644
--- a/zig/src/native/metal.zig
+++ b/zig/src/native/metal.zig
@@ -176,8 +176,8 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c
problem.* = why;
return null;
}
- if (o.drafter != null and !dense and !isGemma4(o.model_type)) {
- problem.* = "--drafter is supported for Qwen3.8-27B-class checkpoints (qwen3_5 with its own output head) and Gemma 4 dense";
+ if (o.drafter != null and !dense and !isGemma4(o.model_type) and !std.mem.eql(u8, o.model_type, "lfm2_moe")) {
+ problem.* = "--drafter is supported for Qwen3.8-27B-class checkpoints (qwen3_5 with its own output head), Gemma 4 dense and LFM2.5-24B-A2B";
return null;
}
if (dense) return openQwen27(a, gpa, io, o, problem);
diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig
index 87914e0042..5f26ed3472 100644
--- a/zig/src/server/cli.zig
+++ b/zig/src/server/cli.zig
@@ -100,7 +100,7 @@ pub const flags = [_]Flag{
.{ .name = "--slide", .kind = .store_true, .native = true },
.{ .name = "--slide-graph", .native = true },
.{ .name = "--drafter", .native = true },
- .{ .name = "--drafter-bits", .choices = &.{ "0", "4" }, .native = true },
+ .{ .name = "--drafter-bits", .choices = &.{ "0", "4", "8" }, .native = true },
.{ .name = "--record", .native = true },
.{ .name = "--kv-cache", .choices = &.{ "bf16", "fp8" }, .native = true },
.{ .name = "--prompt-precision", .choices = &.{ "bf16", "int8" }, .native = true },
@@ -152,7 +152,6 @@ pub const flags = [_]Flag{
// the native server's own: Cortheon's reasoning program and server-run web research for every chat request
.{ .name = "--cortheon", .kind = .store_true, .native = true },
};
-
/// The variables this binary honours as the Python engine does, then the CUDA build's or the Metal build's.
pub const env = [_][]const u8{ "TENSORFOLD_API_KEY", "TENSORFOLD_NO_LIVE", "TENSORFOLD_SEED_SALT", "TENSORFOLD_REQUEST_LOG", "TENSORFOLD_NO_UPDATE_CHECK", "HF_HOME", "HF_HUB_CACHE", "HF_HUB_OFFLINE", "BRAVE_SEARCH_API_KEY" } ++
(if (cuda_build) [_][]const u8{ "TF_CUDA_DEVICE", "TF_CUDA_SEGMENTS", "TENSORFOLD_CUDA_KERNELS", "TENSORFOLD_MEMORY_RESERVE_GIB", "TENSORFOLD_CUDA_MEMORY_LIMIT_GB", "TENSORFOLD_P2P_DIAGNOSTICS", "TENSORFOLD_P2P_PROBE" } else [_][]const u8{}) ++
@@ -217,7 +216,7 @@ pub const Args = struct {
compact_memory: ?[]const u8 = null,
drafter: ?[]const u8 = null,
drafter_name: ?[]const u8 = null, // retained before hub resolution for the engine's serving line
- drafter_bits: u8 = 4,
+ drafter_bits: ?u8 = null, // null: not given (a family without its own default takes 4)
record: ?[]const u8 = null, // drafter training rows written under this folder
kv_cache: []const u8 = "bf16", // fp8: e4m3 attention caches with a scale a row, about twice the context
prompt_precision: []const u8 = "bf16", // int8: prompt matmuls quantize their inputs to int8 a 64-input group
@@ -548,7 +547,8 @@ pub fn request(a: Allocator, dir: []const u8, model_type: []const u8, args: Args
.drafts = !args.no_drafts,
.drafter = args.drafter,
.drafter_name = args.drafter_name orelse args.drafter,
- .drafter_bits = args.drafter_bits,
+ .drafter_bits = args.drafter_bits orelse 4,
+ .drafter_bits_given = args.drafter_bits != null,
.record = args.record,
.kv8 = std.mem.eql(u8, args.kv_cache, "fp8"),
.prompt_int8 = std.mem.eql(u8, args.prompt_precision, "int8"),
diff --git a/zig/src/server/cli_test.zig b/zig/src/server/cli_test.zig
index ca4455cf16..123e4dfe10 100644
--- a/zig/src/server/cli_test.zig
+++ b/zig/src/server/cli_test.zig
@@ -163,8 +163,10 @@ test "parse and capabilities share the table" {
try std.testing.expectEqual(@as(usize, 2), args.api_key.len);
try std.testing.expect(!args.thinking);
try std.testing.expectEqualStrings("x", (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter.?);
- try std.testing.expectEqual(@as(u8, 0), (try parse(a, &.{ "m", "--drafter-bits", "0" }, &u)).drafter_bits);
- try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "8" }, &u));
+ try std.testing.expectEqual(@as(?u8, 0), (try parse(a, &.{ "m", "--drafter-bits", "0" }, &u)).drafter_bits);
+ try std.testing.expectEqual(@as(?u8, 8), (try parse(a, &.{ "m", "--drafter-bits", "8" }, &u)).drafter_bits);
+ try std.testing.expectEqual(@as(?u8, null), (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter_bits);
+ try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "3" }, &u));
try std.testing.expectEqualStrings("/r", (try parse(a, &.{ "m", "--record", "/r" }, &u)).record.?);
try std.testing.expectEqualStrings("fp8", (try parse(a, &.{ "m", "--kv-cache", "fp8" }, &u)).kv_cache);
try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--kv-cache", "int4" }, &u));
diff --git a/zig/src/server/main.zig b/zig/src/server/main.zig
index 7ab94683aa..ab48bd608a 100644
--- a/zig/src/server/main.zig
+++ b/zig/src/server/main.zig
@@ -26,7 +26,7 @@ const usage_line = "usage: tensorfold serve [-h] [--host HOST] [--port PORT] [--
" [--thinking | --no-thinking] [--reasoning-effort {low,medium,high,xhigh}]" ++
" [--thinking-budget THINKING_BUDGET] [--thinking-reserve THINKING_RESERVE] [--loop-guard] [--loop-guard-stop] [--no-drafts]" ++
" [--drafter DRAFTER]" ++
- " [--drafter-bits {0,4}] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++
+ " [--drafter-bits {0,4,8}] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++
" [--silence-floor-dbfs DBFS] [--silence-seconds SECONDS]" ++
" [--silence-onset-s SECONDS] [--silence-onset-voiced-s SECONDS]" ++
" [--silence-mid-s SECONDS] [--silence-mid-min-voiced-s SECONDS] [--silence-redraws N]" ++
diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig
index 4fa4fbe147..cd9d82bc08 100644
--- a/zig/src/tensorfold.zig
+++ b/zig/src/tensorfold.zig
@@ -88,6 +88,10 @@ test {
_ = @import("families/lfm2_moe/config.zig");
_ = @import("families/lfm2_moe/backend.zig");
_ = @import("families/lfm2_moe/state.zig");
+ _ = @import("families/lfm2_moe/dspark_config.zig");
+ _ = @import("families/lfm2_moe/dspark_quant.zig");
+ _ = @import("families/lfm2_moe/dspark_test.zig");
+ _ = @import("families/lfm2_moe/dspark_bf16_test.zig");
}
pub const lane_projection = @import("core/lane_projection.zig");
diff --git a/zig/tests/contract/metal_lfm2_attn.zig b/zig/tests/contract/metal_lfm2_attn.zig
new file mode 100644
index 0000000000..60fa6939ae
--- /dev/null
+++ b/zig/tests/contract/metal_lfm2_attn.zig
@@ -0,0 +1,172 @@
+//! LFM2's block attention (lfm2_attn_block, then lfm2_attn_block_merge): every row of a drafter's block sees all keys.
+const std = @import("std");
+const mtl = @import("metal");
+const contract = @import("contract");
+const sources = @import("kernel_sources");
+const c = contract.spec;
+const g = contract.gates;
+const rig_ = @import("rig.zig");
+const controls = @import("metal_lfm2_controls.zig");
+
+const qh = 32;
+const kvh = 8;
+const hd = 64;
+const subs = 8; // the drafter's sub-spans a 4,096-key chunk (state.subs)
+const chunks = 32;
+const chunk_keys = 4096;
+pub const name = "metal lfm2 block attention";
+
+/// rows: a block's rows; k: the keys every row sees (a context and the block past it).
+const shapes = [_]c.Shape{ .{ .rows = 1, .n = qh * hd, .k = 9 }, .{ .rows = 3, .n = qh * hd, .k = 700 }, .{ .rows = 9, .n = qh * hd, .k = 4500 } };
+
+fn depth(s: c.Shape) usize {
+ return s.k;
+}
+
+pub fn spec() c.Spec {
+ return .{ .name = name, .accum = .none, .depth = depth, .out = .bf16, .served = &shapes, .nan = .{ .stated = "a NaN key of a kv head makes its query heads' outputs NaN, the others stay finite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "the check bounds each output by its bf16 probabilities' rounding, not a sum's bits" } };
+}
+
+fn value(i: usize, seed: u64, scale: f32) u16 {
+ const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40;
+ return c.toBf16((@as(f32, @floatFromInt(h % 2048)) / 1024 - 1) * scale);
+}
+
+pub const Probe = struct {
+ a: std.mem.Allocator,
+ rig: rig_.Rig,
+ mutation: controls.Mutation,
+ pipes: [2]mtl.Pipeline,
+ shape: c.Shape = undefined,
+ cap: usize = 0,
+ nan: bool = false,
+ q: []u16 = &.{},
+ k: []u16 = &.{},
+ v: []u16 = &.{},
+ args: [5]rig_.Ref = undefined,
+ y: rig_.Ref = undefined,
+
+ pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe {
+ const text = try controls.source(a, mutation);
+ defer if (mutation != .none) a.free(text);
+ const first = try rig_.pipeline(device, text, "lfm2_attn_block");
+ errdefer first.deinit();
+ return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipes = .{ first, try rig_.pipeline(device, text, "lfm2_attn_block_merge") }, .mutation = mutation };
+ }
+
+ fn drop(p: *Probe) void {
+ inline for (.{ "q", "k", "v" }) |f| {
+ p.a.free(@field(p, f));
+ @field(p, f) = &.{};
+ }
+ p.rig.reset();
+ }
+
+ pub fn deinit(p: *Probe) void {
+ p.drop();
+ p.rig.deinit();
+ for (p.pipes) |x| x.deinit();
+ }
+
+ const vt = g.hooks(Probe);
+ pub fn kernel(p: *Probe) g.Kernel {
+ var s = spec();
+ if (p.mutation != .none) s.name = p.mutation.name();
+ return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() };
+ }
+
+ pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool {
+ if (s.n != qh * hd or s.rows == 0 or s.rows > 16 or s.k < s.rows or s.k > chunks * chunk_keys) return false;
+ p.drop();
+ p.shape = s;
+ p.nan = f == .nonfinite;
+ p.cap = (s.k + 63) / 64 * 64 + 32; // keys past `k` hold finite values, as a lane's cache does
+ const padded = (s.rows + 7) / 8 * 8 + 8; // the kernel reads a tile's 8 rows whatever its row count
+ p.q = try p.a.alloc(u16, padded * qh * hd);
+ p.k = try p.a.alloc(u16, kvh * p.cap * hd);
+ p.v = try p.a.alloc(u16, kvh * p.cap * hd);
+ for (p.q, 0..) |*x, i| x.* = value(i, seed, 1);
+ for (p.k, 0..) |*x, i| x.* = value(i, seed +% 1, if (f == .stress) 4 else 1);
+ for (p.v, 0..) |*x, i| x.* = value(i, seed +% 2, 2);
+ if (p.nan) p.k[(0 * p.cap + s.k / 2) * hd + 5] = 0x7fc0; // kv head 0, a visible key
+ p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.q));
+ p.args[1] = try p.rig.input(std.mem.sliceAsBytes(p.k));
+ p.args[2] = try p.rig.input(std.mem.sliceAsBytes(p.v));
+ const Args = extern struct { len: u32, cap: u32, scale: f32, subs: u32 };
+ const args = Args{ .len = @intCast(s.k - 1), .cap = @intCast(p.cap), .scale = 1.0 / @sqrt(@as(f32, hd)), .subs = subs };
+ p.args[3] = try p.rig.input(std.mem.asBytes(&args));
+ const n: u32 = @intCast(s.rows);
+ p.args[4] = try p.rig.input(std.mem.asBytes(&n));
+ p.y = try p.rig.output(p.count() * 2);
+ return true;
+ }
+
+ pub fn count(p: *Probe) usize {
+ return p.shape.rows * qh * hd;
+ }
+
+ pub fn run(p: *Probe) !void {
+ const rows = p.shape.rows;
+ const partials = @max(rows, 8) * subs * qh * chunks;
+ const pm = try p.rig.scratch(partials * 4);
+ const pl = try p.rig.scratch(partials * 4);
+ const po = try p.rig.scratch(partials * hd * 4);
+ const pass = p.rig.begin();
+ const e = pass.e;
+ e.setPipeline(p.pipes[0]);
+ for (p.args[0..4], 0..) |r, i| e.setBuffer(r.buf, r.off, i);
+ for ([_]rig_.Ref{ pm, pl, po, p.args[4] }, 4..) |r, i| e.setBuffer(r.buf, r.off, i);
+ const used = (p.shape.k + chunk_keys - 1) / chunk_keys * subs;
+ e.dispatchGroups(mtl.Size.of((rows + 7) / 8, kvh, used), mtl.Size.of(128, 1, 1));
+ e.setPipeline(p.pipes[1]);
+ for ([_]rig_.Ref{ pm, pl, po, p.args[3], p.y }, 0..) |r, i| e.setBuffer(r.buf, r.off, i);
+ e.dispatchGroups(mtl.Size.of(qh, rows, 1), mtl.Size.of(32, 1, 1));
+ try p.rig.finish(pass);
+ }
+
+ pub fn reference(_: *Probe, want: []f64, mag: []f64) !void {
+ @memset(want, 0); // the check computes its own references
+ @memset(mag, 0);
+ }
+
+ /// Every output against float64 attention: within 2^-7 of its values' largest magnitude and 2^-8 of itself.
+ pub fn check(p: *Probe, _: g.Gate) !?[]const u8 {
+ const s = p.shape;
+ const out = std.mem.bytesAsSlice(u16, p.rig.bytes(p.y, p.count() * 2));
+ const w = try p.a.alloc(f64, s.k);
+ defer p.a.free(w);
+ for (0..s.rows) |r| for (0..qh) |h| {
+ const kv = h / (qh / kvh);
+ const nan_head = p.nan and kv == 0;
+ var top: f64 = -std.math.inf(f64);
+ for (0..s.k) |j| {
+ var dot: f64 = 0;
+ for (0..hd) |d| dot += c.fromBf16(p.q[(r * qh + h) * hd + d]) * c.fromBf16(p.k[(kv * p.cap + j) * hd + d]);
+ w[j] = dot / @sqrt(@as(f64, hd));
+ top = @max(top, w[j]);
+ }
+ var sum: f64 = 0;
+ for (w) |*x| {
+ x.* = @exp(x.* - top);
+ sum += x.*;
+ }
+ for (0..hd) |d| {
+ var o: f64 = 0;
+ var big: f64 = 0;
+ for (0..s.k) |j| {
+ const vv: f64 = c.fromBf16(p.v[(kv * p.cap + j) * hd + d]);
+ o += w[j] / sum * vv;
+ big = @max(big, @abs(vv));
+ }
+ const got: f64 = c.fromBf16(out[(r * qh + h) * hd + d]);
+ if (nan_head) {
+ if (!std.math.isNan(got)) return "a head behind a NaN key gave a number";
+ continue;
+ }
+ if (!std.math.isFinite(got)) return "a finite head gave a non-finite output";
+ if (@abs(got - o) > big * 0x1p-7 + @abs(o) * 0x1p-8) return "an output is past its float64 bound";
+ }
+ };
+ return null;
+ }
+};
diff --git a/zig/tests/contract/metal_lfm2_bf16.zig b/zig/tests/contract/metal_lfm2_bf16.zig
new file mode 100644
index 0000000000..fcbe718243
--- /dev/null
+++ b/zig/tests/contract/metal_lfm2_bf16.zig
@@ -0,0 +1,135 @@
+//! LFM2's DSpark bf16 linears under gates 1-4: x W^T with bf16 inputs and fp32 sums, few rows or 64x64 tiles.
+const std = @import("std");
+const mtl = @import("metal");
+const contract = @import("contract");
+const rig_ = @import("rig.zig");
+const controls = @import("metal_lfm2_controls.zig");
+
+const c = contract.spec;
+const g = contract.gates;
+const Dense = contract.dense.Dense;
+
+pub const Which = enum { rows, tile };
+const names = [_][]const u8{ "metal lfm2 bf16 rows", "metal lfm2 bf16 tile" };
+
+/// The drafter's matrices: q and o (2,048 from 2,048), k and v (512), gate and up (6,144), down (2,048 from 6,144).
+const rows_served = [_]c.Shape{
+ .{ .rows = 1, .n = 2048, .k = 2048 }, .{ .rows = 9, .n = 512, .k = 2048 }, .{ .rows = 16, .n = 6144, .k = 2048 },
+ .{ .rows = 32, .n = 2048, .k = 6144 }, .{ .rows = 4, .n = 2048, .k = 10240 },
+};
+/// Shared holds past 16 rows, and a cold prompt's absorb up to 2,048 rows a pass.
+const tile_served = [_]c.Shape{
+ .{ .rows = 17, .n = 2048, .k = 2048 }, .{ .rows = 18, .n = 512, .k = 2048 },
+ .{ .rows = 24, .n = 6144, .k = 2048 }, .{ .rows = 32, .n = 2048, .k = 6144 },
+ .{ .rows = 33, .n = 2048, .k = 2048 }, .{ .rows = 128, .n = 512, .k = 2048 },
+ .{ .rows = 2048, .n = 2048, .k = 10240 }, .{ .rows = 70, .n = 6144, .k = 2048 },
+};
+
+/// A lane's k / 32 fp32 fmas in turn, then the simdgroup's five-level sum.
+fn rowsDepth(s: c.Shape) usize {
+ return s.k / 32 + 8;
+}
+
+/// One 8-wide MMA step a k step of 8, accumulating in fp32.
+fn tileDepth(s: c.Shape) usize {
+ return s.k / 8 + 8;
+}
+
+pub fn spec(w: Which) c.Spec {
+ return switch (w) {
+ // rows in groups of 16, a column a simdgroup (any n), k in 256-wide lane steps of 8
+ .rows => .{ .name = names[0], .accum = .f32, .depth = rowsDepth, .out = .bf16, .tiles = .{ .{ .size = 16 }, .{ .size = 8 }, .{ .size = 256, .step = 8, .min = 8 } }, .served = &rows_served },
+ .tile => .{ .name = names[1], .accum = .f32, .depth = tileDepth, .out = .bf16, .tiles = .{ .{ .size = 64 }, .{ .size = 64 }, .{ .size = 32, .step = 32, .min = 32 } }, .served = &tile_served },
+ };
+}
+
+const Args = extern struct { rows: u32, n: u32, k: u32 };
+
+pub const Probe = struct {
+ a: std.mem.Allocator,
+ rig: rig_.Rig,
+ which: Which,
+ mutation: controls.Mutation,
+ pipe: mtl.Pipeline,
+ d: ?Dense = null,
+ x: rig_.Ref = undefined,
+ w: rig_.Ref = undefined,
+ out: rig_.Ref = undefined,
+
+ pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which, mutation: controls.Mutation) !Probe {
+ const text = try controls.source(a, mutation);
+ defer if (mutation != .none) a.free(text);
+ const key: []const u8 = if (w == .rows) "lfm2_bf16_rows" else "lfm2_bf16_mma";
+ return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .which = w, .mutation = mutation, .pipe = try rig_.pipeline(device, text, key) };
+ }
+
+ pub fn deinit(p: *Probe) void {
+ if (p.d) |d| d.deinit();
+ p.rig.deinit();
+ p.pipe.deinit();
+ }
+
+ const vt = g.hooks(Probe);
+ pub fn kernel(p: *Probe) g.Kernel {
+ var s = spec(p.which);
+ if (p.mutation != .none) s.name = p.mutation.name();
+ return .{ .spec = s, .ctx = p, .vt = &vt, .control = p.mutation.gate() };
+ }
+
+ pub fn prepare(p: *Probe, shape: c.Shape, fill: g.Fill, seed: u64) !bool {
+ if (shape.rows == 0 or shape.n == 0 or shape.k == 0) return false;
+ if (p.which == .rows and (shape.k % 8 != 0 or shape.rows > 32)) return false;
+ if (p.which == .tile and shape.k % 32 != 0) return false;
+ if (p.d) |d| d.deinit();
+ p.d = null;
+ p.rig.reset();
+ const d = try Dense.init(p.a, shape, 1);
+ p.d = d;
+ d.fillX(fill, seed, 2);
+ // W as bf16 bits in [-1, 1), or gate 4's mirrored integers whose halves cancel (partials under 2^24)
+ const bits = try p.a.alloc(u16, shape.n * shape.k);
+ defer p.a.free(bits);
+ var prng = std.Random.DefaultPrng.init(seed ^ 0x6d);
+ const rng = prng.random();
+ const k = shape.k;
+ for (bits) |*b| b.* = c.toBf16(if (fill == .stress) 0 else rng.float(f32) * 2 - 1);
+ if (fill == .stress) for (0..shape.n) |j| for (0..k / 2) |i| {
+ const v: f32 = @floatFromInt(1 + rng.uintLessThan(u32, 7));
+ bits[j * k + i] = c.toBf16(v);
+ bits[j * k + k / 2 + i] = c.toBf16(-v);
+ };
+ for (d.w, bits) |*wv, b| wv.* = c.fromBf16(b);
+ p.x = try p.rig.input(std.mem.sliceAsBytes(d.x));
+ p.w = try p.rig.input(std.mem.sliceAsBytes(bits));
+ p.out = try p.rig.output(d.outputs() * c.size(spec(p.which).out));
+ return true;
+ }
+
+ pub fn run(p: *Probe) !void {
+ const d = p.d.?;
+ const pass = p.rig.begin();
+ const e = pass.e;
+ e.setPipeline(p.pipe);
+ e.setBuffer(p.x.buf, p.x.off, 0);
+ e.setBuffer(p.w.buf, p.w.off, 1);
+ const args = Args{ .rows = @intCast(d.shape.rows), .n = @intCast(d.shape.n), .k = @intCast(d.shape.k) };
+ e.setBytes(std.mem.asBytes(&args), 2);
+ e.setBuffer(p.out.buf, p.out.off, 3);
+ if (p.which == .rows) {
+ e.dispatchGroups(mtl.Size.of((d.shape.n + 7) / 8, (d.shape.rows + 15) / 16, 1), mtl.Size.of(256, 1, 1));
+ } else e.dispatchGroups(mtl.Size.of((d.shape.n + 63) / 64, (d.shape.rows + 63) / 64, 1), mtl.Size.of(128, 1, 1));
+ try p.rig.finish(pass);
+ }
+
+ pub fn count(p: *Probe) usize {
+ return p.d.?.outputs();
+ }
+
+ pub fn reference(p: *Probe, want: []f64, mag: []f64) !void {
+ return p.d.?.reference(want, mag);
+ }
+
+ pub fn simulate(p: *Probe, acc: c.Accum, order: g.Order, out: []f64) !void {
+ return p.d.?.simulate(acc, order, out);
+ }
+};
diff --git a/zig/tests/contract/metal_lfm2_confidence.zig b/zig/tests/contract/metal_lfm2_confidence.zig
new file mode 100644
index 0000000000..ebf74ea5f2
--- /dev/null
+++ b/zig/tests/contract/metal_lfm2_confidence.zig
@@ -0,0 +1,110 @@
+//! LFM2's DSpark confidence logit (lfm2_dspark_confidence): a dot of a row and a W1 row with the head's 2,304 weights.
+const std = @import("std");
+const mtl = @import("metal");
+const contract = @import("contract");
+const sources = @import("kernel_sources");
+const c = contract.spec;
+const g = contract.gates;
+const rows = contract.rows;
+const rig_ = @import("rig.zig");
+const controls = @import("metal_lfm2_controls.zig");
+
+const hidden = 2048;
+const rank = 256;
+pub const name = "metal lfm2 dspark confidence";
+
+/// rows: one stream's drafts (row i reads the block's row i and the W1 row of the token before draft i).
+const shapes = [_]c.Shape{ .{ .rows = 1, .n = 1, .k = hidden + rank }, .{ .rows = 4, .n = 1, .k = hidden + rank }, .{ .rows = 9, .n = 1, .k = hidden + rank } };
+
+/// A thread's chain over every 256th input, a 32-way simd_sum tree, then the eight simdgroups' partials in order.
+fn depth(s: c.Shape) usize {
+ return (s.k + 255) / 256 + 5 + 8;
+}
+
+pub fn spec() c.Spec {
+ return .{ .name = name, .accum = .f32, .depth = depth, .out = .f32, .served = &shapes };
+}
+
+pub const Probe = struct {
+ a: std.mem.Allocator,
+ rig: rig_.Rig,
+ pipe: mtl.Pipeline,
+ mutation: controls.Mutation,
+ r: ?rows.Rows = null,
+ args: [6]rig_.Ref = undefined, // x, W1 rows, ids, picks, the head's weights, the shape
+ out: rig_.Ref = undefined,
+
+ pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe {
+ const text = try controls.source(a, mutation);
+ defer if (mutation != .none) a.free(text);
+ return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, text, "lfm2_dspark_confidence"), .mutation = mutation };
+ }
+
+ pub fn deinit(p: *Probe) void {
+ if (p.r) |r| r.deinit();
+ p.rig.deinit();
+ p.pipe.deinit();
+ }
+
+ const vt = g.hooks(Probe);
+ pub fn kernel(p: *Probe) g.Kernel {
+ var s = spec();
+ if (p.mutation != .none) s.name = p.mutation.name();
+ return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() };
+ }
+
+ /// Row i's first 2,048 inputs as the block's row i, its last 256 as W1 row i: the token before draft i is i.
+ pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool {
+ if (s.k != hidden + rank or s.rows == 0 or s.rows > 16) return false;
+ if (p.r) |r| r.deinit();
+ p.r = null;
+ p.rig.reset();
+ const r = try rows.Rows.init(p.a, .dot, s);
+ p.r = r;
+ r.fill(f, seed);
+ const x = try p.a.alloc(u16, s.rows * hidden);
+ defer p.a.free(x);
+ const w1 = try p.a.alloc(u16, s.rows * rank);
+ defer p.a.free(w1);
+ for (0..s.rows) |i| {
+ @memcpy(x[i * hidden ..][0..hidden], r.x[i * s.k ..][0..hidden]);
+ @memcpy(w1[i * rank ..][0..rank], r.x[i * s.k + hidden ..][0..rank]);
+ }
+ // the block's pending id is token 0; the pick before draft i (i > 0) is token i, a flag word after each
+ var picks: [32]u32 = @splat(0xdead);
+ for (1..s.rows) |i| picks[2 * (i - 1)] = @intCast(i);
+ const ids = [1]u32{0};
+ const args = [5]u32{ @intCast(s.rows), 1, @intCast(s.rows), rank, hidden };
+ p.args[0] = try p.rig.input(std.mem.sliceAsBytes(x));
+ p.args[1] = try p.rig.input(std.mem.sliceAsBytes(w1));
+ p.args[2] = try p.rig.input(std.mem.asBytes(&ids));
+ p.args[3] = try p.rig.input(std.mem.asBytes(&picks));
+ p.args[4] = try p.rig.input(std.mem.sliceAsBytes(r.w));
+ p.args[5] = try p.rig.input(std.mem.asBytes(&args));
+ p.out = try p.rig.output(s.rows * 4);
+ return true;
+ }
+
+ pub fn run(p: *Probe) !void {
+ const n = p.r.?.shape.rows;
+ const pass = p.rig.begin();
+ const e = pass.e;
+ e.setPipeline(p.pipe);
+ for (p.args, 0..) |ref, i| e.setBuffer(ref.buf, ref.off, i);
+ e.setBuffer(p.out.buf, p.out.off, 6);
+ e.dispatchGroups(mtl.Size.of(n, 1, 1), mtl.Size.of(256, 1, 1));
+ try p.rig.finish(pass);
+ }
+
+ pub fn count(p: *Probe) usize {
+ return p.r.?.outputs();
+ }
+
+ pub fn reference(p: *Probe, want: []f64, mag: []f64) !void {
+ return p.r.?.reference(want, mag);
+ }
+
+ pub fn simulate(p: *Probe, acc: c.Accum, order: g.Order, out: []f64) !void {
+ return p.r.?.simulate(acc, order, out);
+ }
+};
diff --git a/zig/tests/contract/metal_lfm2_controls.zig b/zig/tests/contract/metal_lfm2_controls.zig
index 655a97d4fc..bd8a890529 100644
--- a/zig/tests/contract/metal_lfm2_controls.zig
+++ b/zig/tests/contract/metal_lfm2_controls.zig
@@ -9,6 +9,19 @@ pub const Mutation = enum {
argmax_no_flag,
conv_first_state,
attention_first_cache,
+ tap_packed_rows,
+ tap_clobbers_column,
+ markov_nan_dropped,
+ markov_first_prev,
+ draft_ids_unzeroed,
+ block_attention_causal,
+ confidence_no_w1,
+ markov_bf16_nan_dropped,
+ markov_bf16_first_prev,
+ bf16_rows_short_k,
+ bf16_rows_last_row,
+ bf16_tile_untransposed,
+ bf16_tile_last_column,
pub fn name(m: Mutation) []const u8 {
return switch (m) {
@@ -17,6 +30,19 @@ pub const Mutation = enum {
.argmax_no_flag => "control: metal lfm2 argmax, a nonfinite logit is not flagged",
.conv_first_state => "control: metal lfm2 stream conv, every stream reads the first stream's state",
.attention_first_cache => "control: metal lfm2 stream attention, every stream writes the first stream's cache",
+ .tap_packed_rows => "control: metal lfm2 tap copy, rows packed at the hidden size, not the taps' row",
+ .tap_clobbers_column => "control: metal lfm2 tap copy, a column of another tap is overwritten",
+ .markov_nan_dropped => "control: metal lfm2 markov step, a NaN score is not flagged",
+ .markov_first_prev => "control: metal lfm2 markov step, every stream reads the first stream's previous token",
+ .draft_ids_unzeroed => "control: metal lfm2 draft ids, a pick past the vocabulary keeps its token",
+ .confidence_no_w1 => "control: metal lfm2 dspark confidence, the previous token's W1 row is not added",
+ .block_attention_causal => "control: metal lfm2 block attention, a row misses the last key",
+ .markov_bf16_nan_dropped => "control: metal lfm2 markov step bf16, a NaN score is not flagged",
+ .markov_bf16_first_prev => "control: metal lfm2 markov step bf16, every stream reads the first stream's previous token",
+ .bf16_rows_short_k => "control: metal lfm2 bf16 rows, a lane skips every second step of k",
+ .bf16_rows_last_row => "control: metal lfm2 bf16 rows, a group's last row is not computed",
+ .bf16_tile_untransposed => "control: metal lfm2 bf16 tile, W's tile is read untransposed",
+ .bf16_tile_last_column => "control: metal lfm2 bf16 tile, the last column is not written",
};
}
@@ -24,7 +50,7 @@ pub const Mutation = enum {
pub fn gate(m: Mutation) ?contract.gates.Gate {
return switch (m) {
.none => null,
- .argmax_no_flag => .nonfinite,
+ .argmax_no_flag, .markov_nan_dropped, .markov_bf16_nan_dropped => .nonfinite,
else => .shapes,
};
}
@@ -38,6 +64,19 @@ pub const Mutation = enum {
.argmax_no_flag => .{ .from = "if (!isfinite(f)) bad = 1;", .to = "if (false) bad = 1;" },
.conv_first_state => .{ .from = "lfm2_conv_row(bcx + size_t(s.base) * 3 * D, s.conv,", .to = "lfm2_conv_row(bcx + size_t(s.base) * 3 * D, segs[0].conv," },
.attention_first_cache => .{ .from = "queries + b * QH * HD, s.keys, s.values,", .to = "queries + b * QH * HD, segs[0].keys, segs[0].values," },
+ .tap_packed_rows => .{ .from = "(taps + size_t(i.y) * at.x + at.y))[i.x]", .to = "(taps + size_t(i.y) * D + at.y))[i.x]" },
+ .markov_nan_dropped => .{ .from = "nan = nan || isnan(sc);", .to = "nan = false;" },
+ .markov_first_prev => .{ .from = "id = prev[size_t(s) * p.prev_stride];", .to = "id = prev[0];" },
+ .draft_ids_unzeroed => .{ .from = "ids[i] = ok ? token : 0u;", .to = "ids[i] = token;" },
+ .confidence_no_w1 => .{ .from = "if (id < p.vocab)\n for (uint k = t; k < p.rank", .to = "if (false)\n for (uint k = t; k < p.rank" },
+ .block_attention_causal => .{ .from = "count[b] = FULL ? p.len + 1 :", .to = "count[b] = FULL ? p.len :" },
+ .markov_bf16_nan_dropped => .{ .from = "nan = nan || isnan(sc);", .to = "nan = false;" },
+ .markov_bf16_first_prev => .{ .from = "id = prev[size_t(s) * p.prev_stride];", .to = "id = prev[0];" },
+ .bf16_rows_short_k => .{ .from = "for (uint i = 8 * lane; i < p.k; i += 256) {", .to = "for (uint i = 8 * lane; i < p.k; i += 512) {" },
+ .bf16_rows_last_row => .{ .from = "if (r >= rows) break;", .to = "if (r + 1 >= rows) break;" },
+ .bf16_tile_untransposed => .{ .from = "32, ulong2(0, 0), true);", .to = "32, ulong2(0, 0), false);" },
+ .bf16_tile_last_column => .{ .from = "if (row < p.rows && col < p.n) y[", .to = "if (row < p.rows && col + 1 < p.n) y[" },
+ .tap_clobbers_column => .{ .from = "[i.x] = from[i.x];\n}\n", .to = "[i.x] = from[i.x];\n *(device bfloat*)(taps + size_t(i.y) * at.x) = 0;\n}\n" },
};
}
};
diff --git a/zig/tests/contract/metal_lfm2_dspark.zig b/zig/tests/contract/metal_lfm2_dspark.zig
new file mode 100644
index 0000000000..04b7fc489a
--- /dev/null
+++ b/zig/tests/contract/metal_lfm2_dspark.zig
@@ -0,0 +1,264 @@
+//! LFM2's DSpark kernels: the Markov step (scan then pick), and held drafts into a verify's ids.
+const std = @import("std");
+const mtl = @import("metal");
+const contract = @import("contract");
+const sources = @import("kernel_sources");
+const c = contract.spec;
+const g = contract.gates;
+const rig_ = @import("rig.zig");
+const controls = @import("metal_lfm2_controls.zig");
+
+pub const Which = enum { markov_step, draft_ids, markov_step_bf16 };
+const rank = 256;
+const vocab = 128000;
+const span = 512; // tokens a scan threadgroup takes (the kernel's MARKOV_SPAN)
+const groups = (vocab + span - 1) / span;
+const table = 1000; // W1's rows in the check: ids are inside it or past the vocabulary (a zero row)
+const names = [_][]const u8{ "metal lfm2 markov step", "metal lfm2 draft ids", "metal lfm2 markov step bf16" };
+const stride = 64; // words between a stream's picks: a hold pass's 32 blocks, a token and a flag each
+
+const ids_shapes = [_]c.Shape{ .{ .rows = 1, .n = 1, .k = 1 }, .{ .rows = 4, .n = 1, .k = 1 }, .{ .rows = 15, .n = 1, .k = 1 } };
+const step_shapes = [_]c.Shape{ .{ .rows = 1, .n = vocab, .k = rank }, .{ .rows = 4, .n = vocab, .k = rank }, .{ .rows = 9, .n = vocab, .k = rank } };
+
+fn none(_: c.Shape) usize {
+ return 0;
+}
+
+fn depth(_: c.Shape) usize {
+ return rank;
+}
+
+pub fn spec(w: Which) c.Spec {
+ return switch (w) {
+ .draft_ids => .{ .name = names[1], .accum = .none, .depth = none, .out = .raw, .served = &ids_shapes, .sums = .{ .f16 = 0, .bf16 = 0, .why = "a copy moves tokens: no sums" } },
+ .markov_step, .markov_step_bf16 => .{ .name = if (w == .markov_step) names[0] else names[2], .accum = .none, .depth = depth, .out = .raw, .served = &step_shapes, .nan = .{ .stated = "a row with a NaN score is flagged nonfinite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "picks are tokens: the check bounds each pick's score by gamma(256), not sum bits" } },
+ };
+}
+
+/// A bf16 value for element `i` over signs and binades; the nonfinite fill plants NaN and infinities.
+fn value(i: usize, seed: u64, fill: g.Fill) u16 {
+ if (fill == .nonfinite) switch ((@as(u64, i) *% 2654435761 +% seed) % 9973) {
+ 0 => return 0x7fc0,
+ 1 => return 0x7f80,
+ 2 => return 0xff80,
+ else => {},
+ };
+ const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40;
+ const mant: f32 = @as(f32, @floatFromInt(h % 1024)) / 1024 + 1;
+ const exp: i32 = @as(i32, @intCast((h >> 10) % 8)) - 4;
+ const v = std.math.ldexp(mant, exp) * @as(f32, if (((h >> 15) & 1) == 0) 1 else -1);
+ return c.toBf16(v);
+}
+
+pub const Probe = struct {
+ a: std.mem.Allocator,
+ rig: rig_.Rig,
+ which: Which,
+ mutation: controls.Mutation,
+ pipes: [2]mtl.Pipeline,
+ rows: usize = 0,
+ src: []u16 = &.{}, // the base logits rows
+ w1: []u16 = &.{},
+ codes: []u8 = &.{},
+ scales: []u16 = &.{},
+ biases: []u16 = &.{},
+ w2: []u16 = &.{}, // the bf16 step's W2
+ ids: []u32 = &.{},
+ args: [8]rig_.Ref = undefined,
+ y: rig_.Ref = undefined,
+ bad: rig_.Ref = undefined, // draft_ids: the flag word it sets for a flagged or invalid pick, zeroed each run
+
+ pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which, mutation: controls.Mutation) !Probe {
+ const text = try controls.source(a, mutation);
+ defer if (mutation != .none) a.free(text);
+ const kernels: [2][:0]const u8 = switch (w) {
+ .markov_step => .{ "lfm2_markov_scan", "lfm2_markov_pick" },
+ .markov_step_bf16 => .{ "lfm2_markov_scan_bf16", "lfm2_markov_pick" },
+ .draft_ids => .{ "lfm2_draft_ids", "lfm2_draft_ids" },
+ };
+ const first = try rig_.pipeline(device, text, kernels[0]);
+ errdefer first.deinit();
+ const second = try rig_.pipeline(device, text, kernels[1]);
+ return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .which = w, .mutation = mutation, .pipes = .{ first, second } };
+ }
+
+ fn drop(p: *Probe) void {
+ inline for (.{ "src", "w1", "codes", "scales", "biases", "w2", "ids" }) |f| {
+ p.a.free(@field(p, f));
+ @field(p, f) = &.{};
+ }
+ p.rig.reset();
+ }
+
+ pub fn deinit(p: *Probe) void {
+ p.drop();
+ p.rig.deinit();
+ for (p.pipes) |x| x.deinit();
+ }
+
+ const vt = g.hooks(Probe);
+ pub fn kernel(p: *Probe) g.Kernel {
+ var s = spec(p.which);
+ if (p.mutation != .none) s.name = p.mutation.name();
+ return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() };
+ }
+
+ fn fill(out: []u16, seed: u64, f: g.Fill) void {
+ for (out, 0..) |*v, i| v.* = value(i, seed, f);
+ }
+
+ pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool {
+ p.drop();
+ p.rows = s.rows;
+ if (p.which == .draft_ids) { // picks a stride apart: tokens inside the vocabulary, its last, and past it
+ if (s.n != 1 or s.rows == 0 or s.rows > 16) return false;
+ p.ids = try p.a.alloc(u32, s.rows * stride);
+ for (p.ids, 0..) |*id, i| id.* = switch ((i / stride) % 3) {
+ 0 => @intCast((@as(u64, i) * 7919 + seed) % vocab),
+ 1 => vocab - 1,
+ else => vocab + @as(u32, @intCast(i % 5)),
+ };
+ // the flag word after each pick: zero, but planted nonzero on the last of every four streams
+ for (0..s.rows) |r| p.ids[r * stride + 1] = if (r % 4 == 3) 1 else 0;
+ p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.ids));
+ const at = [4]u32{ stride, @intCast(s.rows), vocab, 0 };
+ p.args[1] = try p.rig.input(std.mem.asBytes(&at));
+ p.y = try p.rig.output(p.count());
+ return true;
+ }
+ if (s.n != vocab or s.k != rank or s.rows == 0) return false;
+ // each stream's step row sits a block (two rows here) apart, as a hold's rows of one step do
+ p.src = try p.a.alloc(u16, 2 * s.rows * vocab);
+ fill(p.src, seed, f);
+ p.w1 = try p.a.alloc(u16, table * rank);
+ fill(p.w1, seed +% 5, .random);
+ p.codes = try p.a.alloc(u8, vocab * rank);
+ for (p.codes, 0..) |*q, i| q.* = @truncate((@as(u64, i) *% 2654435761 +% seed) >> 11);
+ p.scales = try p.a.alloc(u16, vocab * rank / 64);
+ p.biases = try p.a.alloc(u16, vocab * rank / 64);
+ for (p.scales, p.biases, 0..) |*sc, *bi, i| {
+ sc.* = c.toBf16(@as(f32, @floatFromInt(i % 7 + 1)) / 4096);
+ bi.* = c.toBf16(@as(f32, @floatFromInt(i % 5)) / 64 - 0.03);
+ }
+ if (p.which == .markov_step_bf16) {
+ p.w2 = try p.a.alloc(u16, vocab * rank);
+ fill(p.w2, seed +% 9, .random);
+ }
+ // the previous token every second word (the picks' layout): inside W1, its last row, or past the vocabulary
+ p.ids = try p.a.alloc(u32, 2 * s.rows);
+ for (p.ids, 0..) |*id, i| id.* = if (i % 2 == 1) 0xdead else switch ((i / 2) % 3) {
+ 0 => @intCast((@as(u64, i) * 37 + seed) % table),
+ 1 => table - 1,
+ else => vocab + @as(u32, @intCast(i)),
+ };
+ p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.src));
+ p.args[1] = try p.rig.input(std.mem.sliceAsBytes(p.w1));
+ p.args[2] = if (p.which == .markov_step_bf16) try p.rig.input(std.mem.sliceAsBytes(p.w2)) else try p.rig.input(p.codes);
+ p.args[3] = try p.rig.input(std.mem.sliceAsBytes(p.ids));
+ const args = [4]u32{ 2 * vocab, rank, vocab, 2 };
+ p.args[4] = try p.rig.input(std.mem.asBytes(&args));
+ p.args[5] = try p.rig.input(std.mem.sliceAsBytes(p.scales));
+ p.args[6] = try p.rig.input(std.mem.sliceAsBytes(p.biases));
+ const n: u32 = groups;
+ p.args[7] = try p.rig.input(std.mem.asBytes(&n));
+ p.y = try p.rig.output(p.count());
+ return true;
+ }
+
+ pub fn count(p: *Probe) usize {
+ return switch (p.which) {
+ .markov_step, .markov_step_bf16 => p.rows * 8,
+ .draft_ids => p.rows * 4,
+ };
+ }
+
+ pub fn run(p: *Probe) !void {
+ const pass = p.rig.begin();
+ const e = pass.e;
+ if (p.which == .draft_ids) {
+ e.setPipeline(p.pipes[0]);
+ e.setBuffer(p.args[0].buf, p.args[0].off, 0);
+ e.setBuffer(p.args[1].buf, p.args[1].off, 1);
+ e.setBuffer(p.y.buf, p.y.off, 2);
+ p.bad = try p.rig.scratch(4);
+ @memset(p.rig.bytes(p.bad, 4), 0);
+ e.setBuffer(p.bad.buf, p.bad.off, 3);
+ e.dispatchThreads(mtl.Size.of(p.rows, 1, 1), mtl.Size.of(32, 1, 1));
+ return p.rig.finish(pass);
+ }
+ const best = try p.rig.scratch(p.rows * groups * 4);
+ const at = try p.rig.scratch(p.rows * groups * 4);
+ e.setPipeline(p.pipes[0]);
+ for (p.args[0..5], 0..) |r, i| e.setBuffer(r.buf, r.off, i);
+ e.setBuffer(best.buf, best.off, 5);
+ e.setBuffer(at.buf, at.off, 6);
+ if (p.which == .markov_step) {
+ e.setBuffer(p.args[5].buf, p.args[5].off, 7);
+ e.setBuffer(p.args[6].buf, p.args[6].off, 8);
+ }
+ e.dispatchGroups(mtl.Size.of(groups, p.rows, 1), mtl.Size.of(256, 1, 1));
+ e.setPipeline(p.pipes[1]);
+ e.setBuffer(best.buf, best.off, 0);
+ e.setBuffer(at.buf, at.off, 1);
+ e.setBuffer(p.args[7].buf, p.args[7].off, 2);
+ e.setBuffer(p.y.buf, p.y.off, 3);
+ e.dispatchGroups(mtl.Size.of(p.rows, 1, 1), mtl.Size.of(256, 1, 1));
+ try p.rig.finish(pass);
+ }
+
+ pub fn reference(_: *Probe, want: []f64, mag: []f64) !void {
+ @memset(mag, 0);
+ @memset(want, 0); // the step and the ids check themselves (check)
+ }
+
+ /// Each stream's pick against float64 scores with the kernel's bf16 roundings: the best, but for near ties.
+ pub fn check(p: *Probe, _: g.Gate) !?[]const u8 {
+ const out = p.rig.bytes(p.y, p.count());
+ if (p.which == .draft_ids) { // exact: each stream's first pick, or 0 for a token past the vocabulary or a flag
+ var any = false;
+ for (0..p.rows) |r| {
+ const token = p.ids[r * stride];
+ const ok = token < vocab and p.ids[r * stride + 1] == 0;
+ any = any or !ok;
+ if (std.mem.readInt(u32, out[4 * r ..][0..4], .little) != (if (ok) token else 0)) return "an id differs from its pick";
+ }
+ const raised = std.mem.readInt(u32, p.rig.bytes(p.bad, 4)[0..4], .little);
+ if ((raised != 0) != any) return "the bad flag does not follow the flagged and invalid picks";
+ return null;
+ }
+ for (0..p.rows) |s| {
+ const token = std.mem.readInt(u32, out[8 * s ..][0..4], .little);
+ const flag = std.mem.readInt(u32, out[8 * s + 4 ..][0..4], .little);
+ const id = p.ids[2 * s];
+ var top: f64 = -std.math.inf(f64);
+ var mine: ?f64 = null;
+ var nan = false;
+ var bound: f64 = 0;
+ for (0..vocab) |v| {
+ var dot: f64 = 0;
+ var size: f64 = 0;
+ if (id < table) for (0..rank) |i| {
+ const k = (v * rank + i) / 64;
+ const w: f64 = if (p.which == .markov_step_bf16) c.fromBf16(p.w2[v * rank + i]) else @as(f64, c.fromBf16(p.scales[k])) * @as(f64, @floatFromInt(p.codes[v * rank + i])) + c.fromBf16(p.biases[k]);
+ dot += w * c.fromBf16(p.w1[id * rank + i]);
+ size += @abs(w * c.fromBf16(p.w1[id * rank + i]));
+ };
+ const base: f32 = c.fromBf16(p.src[2 * s * vocab + v]);
+ const score: f64 = c.fromBf16(c.toBf16(base + c.fromBf16(c.toBf16(@floatCast(dot)))));
+ if (std.math.isNan(score)) nan = true;
+ if (score > top) top = score;
+ if (v == token) mine = score;
+ // two bf16 steps of the score and the fp32 sum's own error
+ bound = @max(bound, @abs(score) * 0x1p-7 + c.gamma(rank, 0x1p-24) * size);
+ }
+ if (nan) {
+ if (flag != 1) return "a row with a NaN score is not flagged";
+ continue;
+ }
+ if (flag != 0) return "a finite row is flagged nonfinite";
+ const got = mine orelse return "the pick is past the vocabulary";
+ if (got < top - bound) return "the pick's score is below the row's best by more than its bound";
+ }
+ return null;
+ }
+};
diff --git a/zig/tests/contract/metal_lfm2_tap.zig b/zig/tests/contract/metal_lfm2_tap.zig
new file mode 100644
index 0000000000..021baffc9e
--- /dev/null
+++ b/zig/tests/contract/metal_lfm2_tap.zig
@@ -0,0 +1,137 @@
+//! LFM2's tap copy for an external drafter (lfm2_tap): a layer's rows into their tap's columns, the others kept.
+const std = @import("std");
+const mtl = @import("metal");
+const contract = @import("contract");
+const sources = @import("kernel_sources");
+const c = contract.spec;
+const g = contract.gates;
+const rig_ = @import("rig.zig");
+const controls = @import("metal_lfm2_controls.zig");
+
+const hidden = 2048;
+const taps = 5;
+pub const name = "metal lfm2 tap copy";
+
+/// rows: one decode row, a verify window, a shared round's 32 and a 2,048-row prompt chunk.
+const shapes = [_]c.Shape{ .{ .rows = 1, .n = taps * hidden, .k = 1 }, .{ .rows = 10, .n = taps * hidden, .k = 1 }, .{ .rows = 32, .n = taps * hidden, .k = 1 }, .{ .rows = 2048, .n = taps * hidden, .k = 1 } };
+
+fn none(_: c.Shape) usize {
+ return 0;
+}
+
+pub fn spec() c.Spec {
+ return .{ .name = name, .accum = .none, .depth = none, .out = .bf16, .served = &shapes, .sums = .{ .f16 = 0, .bf16 = 0, .why = "a copy moves values: no sums" } };
+}
+
+/// A bf16 value for element `i` over signs and binades; the nonfinite fill plants NaN and infinities.
+fn value(i: usize, seed: u64, fill: g.Fill) u16 {
+ if (fill == .nonfinite) switch ((@as(u64, i) *% 2654435761 +% seed) % 9973) {
+ 0 => return 0x7fc0,
+ 1 => return 0x7f80,
+ 2 => return 0xff80,
+ else => {},
+ };
+ const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40;
+ const mant: f32 = @as(f32, @floatFromInt(h % 1024)) / 1024 + 1;
+ const exp: i32 = @as(i32, @intCast((h >> 10) % 8)) - 4;
+ const v = std.math.ldexp(mant, exp) * @as(f32, if (((h >> 15) & 1) == 0) 1 else -1);
+ return c.toBf16(v);
+}
+
+pub const Probe = struct {
+ a: std.mem.Allocator,
+ rig: rig_.Rig,
+ pipe: mtl.Pipeline,
+ mutation: controls.Mutation,
+ rows: usize = 0,
+ src: []u16 = &.{}, // H rows
+ prior: []u16 = &.{}, // the taps rows as the call finds them (the other taps' columns must stay)
+ args: [2]rig_.Ref = undefined,
+ y: rig_.Ref = undefined,
+
+ pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe {
+ const text = try controls.source(a, mutation);
+ defer if (mutation != .none) a.free(text);
+ return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, text, "lfm2_tap"), .mutation = mutation };
+ }
+
+ fn drop(p: *Probe) void {
+ p.a.free(p.src);
+ p.a.free(p.prior);
+ p.src = &.{};
+ p.prior = &.{};
+ p.rig.reset();
+ }
+
+ pub fn deinit(p: *Probe) void {
+ p.drop();
+ p.rig.deinit();
+ p.pipe.deinit();
+ }
+
+ const vt = g.hooks(Probe);
+ pub fn kernel(p: *Probe) g.Kernel {
+ var s = spec();
+ if (p.mutation != .none) s.name = p.mutation.name();
+ return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() };
+ }
+
+ fn fill(out: []u16, seed: u64, f: g.Fill) void {
+ for (out, 0..) |*v, i| v.* = value(i, seed, f);
+ }
+
+ pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool {
+ p.drop();
+ if (s.n != taps * hidden or s.rows == 0) return false;
+ p.rows = s.rows;
+ p.src = try p.a.alloc(u16, s.rows * hidden);
+ fill(p.src, seed, f);
+ p.prior = try p.a.alloc(u16, s.rows * taps * hidden);
+ fill(p.prior, seed +% 3, .random);
+ p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.src));
+ const at = [2]u32{ taps * hidden, 2 * hidden }; // the third tap's columns
+ p.args[1] = try p.rig.input(std.mem.asBytes(&at));
+ p.y = try p.rig.output(p.count() * 2);
+ return true;
+ }
+
+ pub fn count(p: *Probe) usize {
+ return p.rows * taps * hidden;
+ }
+
+ pub fn run(p: *Probe) !void {
+ @memcpy(p.rig.bytes(p.y, p.count() * 2), std.mem.sliceAsBytes(p.prior));
+ const pass = p.rig.begin();
+ const e = pass.e;
+ e.setPipeline(p.pipe);
+ e.setBuffer(p.args[0].buf, p.args[0].off, 0);
+ e.setBuffer(p.y.buf, p.y.off, 1);
+ e.setBuffer(p.args[1].buf, p.args[1].off, 2);
+ e.dispatchThreads(mtl.Size.of(hidden / 8, p.rows, 1), mtl.Size.of(256, 1, 1));
+ return p.rig.finish(pass);
+ }
+
+ pub fn reference(p: *Probe, want: []f64, mag: []f64) !void {
+ for (want, mag, p.prior) |*w, *m, v| { // the other taps' columns keep what the call found
+ w.* = c.fromBf16(v);
+ m.* = @abs(w.*);
+ }
+ for (0..p.rows) |r| for (0..hidden) |i| {
+ const v = c.fromBf16(p.src[r * hidden + i]);
+ want[r * taps * hidden + 2 * hidden + i] = v;
+ mag[r * taps * hidden + 2 * hidden + i] = @abs(v);
+ };
+ }
+
+ /// The copy is exact: the third tap's columns hold H's bits, the others what the call found.
+ pub fn check(p: *Probe, _: g.Gate) !?[]const u8 {
+ const out = std.mem.bytesAsSlice(u16, p.rig.bytes(p.y, p.count() * 2));
+ for (out, p.prior, 0..) |got, before, i| {
+ const r = i / (taps * hidden);
+ const col = i % (taps * hidden);
+ const want = if (col >= 2 * hidden and col < 3 * hidden) p.src[r * hidden + col - 2 * hidden] else before;
+ if (got != want) return "an output differs from the copy's bits";
+ }
+ return null;
+ }
+};
diff --git a/zig/tests/contract/metal_main.zig b/zig/tests/contract/metal_main.zig
index 46801e171c..52a0d205c3 100644
--- a/zig/tests/contract/metal_main.zig
+++ b/zig/tests/contract/metal_main.zig
@@ -27,6 +27,11 @@ const gemma_mixed = @import("metal_gemma4_mixed.zig");
const lfm2_argmax = @import("metal_lfm2_argmax.zig");
const lfm2_streams = @import("metal_lfm2_streams.zig");
const lfm2_controls = @import("metal_lfm2_controls.zig");
+const lfm2_tap = @import("metal_lfm2_tap.zig");
+const lfm2_dspark = @import("metal_lfm2_dspark.zig");
+const lfm2_bf16 = @import("metal_lfm2_bf16.zig");
+const lfm2_attn = @import("metal_lfm2_attn.zig");
+const lfm2_conf = @import("metal_lfm2_confidence.zig");
const g = contract.gates;
const usage = "usage: tf-kernel-contract [--only TEXT] [--no-gates] [--no-limits] [--offline DIR] [--seed N] [--repro] [--rotary-long] [--rotary-accuracy FILE]";
@@ -191,6 +196,101 @@ pub fn main(init: std.process.Init) !u8 {
try kernels.append(a, p.kernel());
}
}
+ var tap_list: std.ArrayList(*lfm2_tap.Probe) = .empty;
+ defer {
+ for (tap_list.items) |p| {
+ p.deinit();
+ a.destroy(p);
+ }
+ tap_list.deinit(a);
+ }
+ for ([_]lfm2_controls.Mutation{ .none, .tap_packed_rows, .tap_clobbers_column }) |m| {
+ if (!wanted(o, if (m == .none) lfm2_tap.name else m.name())) continue;
+ const p = try a.create(lfm2_tap.Probe);
+ errdefer a.destroy(p);
+ p.* = try lfm2_tap.Probe.init(a, device, queue, m);
+ try tap_list.append(a, p);
+ try kernels.append(a, p.kernel());
+ }
+ var drafter_list: std.ArrayList(*lfm2_dspark.Probe) = .empty;
+ var block_list: std.ArrayList(*lfm2_attn.Probe) = .empty;
+ var bf16_list: std.ArrayList(*lfm2_bf16.Probe) = .empty;
+ defer {
+ for (drafter_list.items) |p| {
+ p.deinit();
+ a.destroy(p);
+ }
+ for (block_list.items) |p| {
+ p.deinit();
+ a.destroy(p);
+ }
+ for (bf16_list.items) |p| {
+ p.deinit();
+ a.destroy(p);
+ }
+ drafter_list.deinit(a);
+ block_list.deinit(a);
+ bf16_list.deinit(a);
+ }
+ for (std.enums.values(lfm2_controls.Mutation)) |m| {
+ const drafter_which: ?lfm2_dspark.Which = switch (m) {
+ .markov_nan_dropped, .markov_first_prev => .markov_step,
+ .markov_bf16_nan_dropped, .markov_bf16_first_prev => .markov_step_bf16,
+ .draft_ids_unzeroed => .draft_ids,
+ else => null,
+ };
+ if (m == .none or drafter_which != null) {
+ for (std.enums.values(lfm2_dspark.Which)) |w| {
+ if (m != .none and w != drafter_which.?) continue;
+ if (!wanted(o, if (m == .none) lfm2_dspark.spec(w).name else m.name())) continue;
+ const p = try a.create(lfm2_dspark.Probe);
+ errdefer a.destroy(p);
+ p.* = try lfm2_dspark.Probe.init(a, device, queue, w, m);
+ try drafter_list.append(a, p);
+ try kernels.append(a, p.kernel());
+ }
+ }
+ const bf16_which: ?lfm2_bf16.Which = switch (m) {
+ .bf16_rows_short_k, .bf16_rows_last_row => .rows,
+ .bf16_tile_untransposed, .bf16_tile_last_column => .tile,
+ else => null,
+ };
+ if (m == .none or bf16_which != null) {
+ for (std.enums.values(lfm2_bf16.Which)) |w| {
+ if (m != .none and w != bf16_which.?) continue;
+ if (!wanted(o, if (m == .none) lfm2_bf16.spec(w).name else m.name())) continue;
+ const p = try a.create(lfm2_bf16.Probe);
+ errdefer a.destroy(p);
+ p.* = try lfm2_bf16.Probe.init(a, device, queue, w, m);
+ try bf16_list.append(a, p);
+ try kernels.append(a, p.kernel());
+ }
+ }
+ if (m == .none or m == .block_attention_causal) {
+ if (!wanted(o, if (m == .none) lfm2_attn.name else m.name())) continue;
+ const p = try a.create(lfm2_attn.Probe);
+ errdefer a.destroy(p);
+ p.* = try lfm2_attn.Probe.init(a, device, queue, m);
+ try block_list.append(a, p);
+ try kernels.append(a, p.kernel());
+ }
+ }
+ var conf_list: std.ArrayList(*lfm2_conf.Probe) = .empty;
+ defer {
+ for (conf_list.items) |p| {
+ p.deinit();
+ a.destroy(p);
+ }
+ conf_list.deinit(a);
+ }
+ for ([_]lfm2_controls.Mutation{ .none, .confidence_no_w1 }) |m| {
+ if (!wanted(o, if (m == .none) lfm2_conf.name else m.name())) continue;
+ const p = try a.create(lfm2_conf.Probe);
+ errdefer a.destroy(p);
+ p.* = try lfm2_conf.Probe.init(a, device, queue, m);
+ try conf_list.append(a, p);
+ try kernels.append(a, p.kernel());
+ }
var rows_list: std.ArrayList(*rows_k.Row) = .empty;
var proj_list: std.ArrayList(*affine_k.Proj) = .empty;
defer for (rows_list.items) |r| r.deinit();
diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig
new file mode 100644
index 0000000000..4fb630fb13
--- /dev/null
+++ b/zig/tests/lfm2_dspark.zig
@@ -0,0 +1,398 @@
+//! LFM2's DSpark drafter on the GPU against an fp32 host reference of the released model, on the real weights.
+const std = @import("std");
+const mtl = @import("metal");
+const tf = @import("tensorfold");
+const q = tf.lfm2_moe;
+const c = q.config;
+const st = q.state;
+const ck = tf.checkpoint;
+const quant = q.dspark_quant;
+
+const hd = c.head_dim;
+const kw = c.kv_heads * hd;
+const qw = c.query_heads * hd;
+
+/// The drafter's tensors as fp32 on the host, in the checkpoint's own layout (interleaved RoPE pairs).
+const Ref = struct {
+ gpa: std.mem.Allocator,
+ check: ck.Checkpoint,
+ cfg: q.dspark_config.Config,
+
+ fn get(r: *const Ref, name: []const u8) ![]f32 {
+ const t = try r.check.get(name);
+ const src = t.host(u16);
+ const out = try r.gpa.alloc(f32, src.len);
+ for (out, src) |*o, v| o.* = quant.toF32(v);
+ return out;
+ }
+
+ fn layer(r: *const Ref, buf: []u8, i: usize, part: []const u8) ![]f32 {
+ return r.get(try std.fmt.bufPrint(buf, "layers.{d}.{s}.weight", .{ i, part }));
+ }
+};
+
+/// y[n] = W[n, k] x[k] for each of `rows` rows, on every core.
+fn matmul(gpa: std.mem.Allocator, w: []const f32, x: []const f32, y: []f32, n: usize, k: usize, rows: usize) !void {
+ const Job = struct {
+ fn run(w_: []const f32, x_: []const f32, y_: []f32, n_: usize, k_: usize, rows_: usize, from: usize, to: usize) void {
+ for (0..rows_) |r| for (from..to) |col| {
+ var s: f32 = 0;
+ const wr = w_[col * k_ ..][0..k_];
+ const xr = x_[r * k_ ..][0..k_];
+ for (wr, xr) |a, b| s += a * b;
+ y_[r * n_ + col] = s;
+ };
+ }
+ };
+ const threads = @min(std.Thread.getCpuCount() catch 1, 16);
+ const per = (n + threads - 1) / threads;
+ const hs = try gpa.alloc(std.Thread, threads);
+ defer gpa.free(hs);
+ var made: usize = 0;
+ defer for (hs[0..made]) |h| h.join(); // a failed spawn still joins the workers already running
+ var at: usize = 0;
+ while (at < n) : (at += per) {
+ hs[made] = try std.Thread.spawn(.{}, Job.run, .{ w, x, y, n, k, rows, at, @min(n, at + per) });
+ made += 1;
+ }
+}
+
+fn rms(x: []f32, gain: []const f32) void {
+ var ss: f32 = 0;
+ for (x) |v| ss += v * v;
+ const inv = 1 / @sqrt(ss / @as(f32, @floatFromInt(x.len)) + c.eps);
+ for (x, gain) |*v, g| v.* = v.* * inv * g;
+}
+
+/// Interleaved RoPE at `pos` over each head of `x` (pairs 2i, 2i + 1).
+fn rope(x: []f32, pos: u64, theta: f32) void {
+ var h: usize = 0;
+ while (h < x.len) : (h += hd) for (0..hd / 2) |i| {
+ const angle = @as(f32, @floatFromInt(pos)) * std.math.pow(f32, theta, -@as(f32, @floatFromInt(i)) / (hd / 2));
+ const a = x[h + 2 * i];
+ const b = x[h + 2 * i + 1];
+ x[h + 2 * i] = a * @cos(angle) - b * @sin(angle);
+ x[h + 2 * i + 1] = a * @sin(angle) + b * @cos(angle);
+ };
+}
+
+/// The target's embedding row `id`, dequantized from its codes at the checkpoint's own width (scale * q + bias).
+fn embedRow(m: *q.Model, id: u32, out: []f32) void {
+ const e = m.weights.embedding;
+ const bits: usize = e.bits;
+ const row_bytes = c.hidden * bits / 8;
+ const codes = (e.w.contents() + e.w_off)[@as(usize, id) * row_bytes ..][0..row_bytes];
+ const sc = @as([*]const u16, @ptrCast(@alignCast(e.scales.contents() + e.s_off)))[@as(usize, id) * c.hidden / 64 ..][0 .. c.hidden / 64];
+ const bi = @as([*]const u16, @ptrCast(@alignCast(e.biases.contents() + e.b_off)))[@as(usize, id) * c.hidden / 64 ..][0 .. c.hidden / 64];
+ for (0..c.hidden) |j| {
+ // codes are a little-endian bit stream: code j takes bits [j * bits, (j + 1) * bits)
+ const at = j * bits;
+ var word: u32 = codes[at / 8];
+ if (at / 8 + 1 < row_bytes) word |= @as(u32, codes[at / 8 + 1]) << 8;
+ const qv = (word >> @intCast(at % 8)) & ((@as(u32, 1) << @intCast(bits)) - 1);
+ out[j] = quant.toF32(sc[j / 64]) * @as(f32, @floatFromInt(qv)) + quant.toF32(bi[j / 64]);
+ }
+}
+
+/// GPU bf16 row against the reference's row + bias: cosine and worst difference.
+fn compare(label: []const u8, gpu: []const u16, row: []const f32, bias: []const f32) void {
+ var dot: f64 = 0;
+ var gg: f64 = 0;
+ var rr: f64 = 0;
+ var worst: f32 = 0;
+ for (gpu, row, bias) |g, b, m2| {
+ const x = quant.toF32(g);
+ const y = b + m2;
+ dot += x * y;
+ gg += x * x;
+ rr += y * y;
+ worst = @max(worst, @abs(x - y));
+ }
+ std.debug.print("{s}: cosine {d:.5}, worst difference {d:.3}\n", .{ label, dot / @sqrt(gg * rr), worst });
+}
+
+/// A hold's GPU ms: the blocks held in a signalled command buffer, waited on and read for its GPU times.
+fn gpuHold(d: *q.dspark.Drafter, blocks: []const q.dspark_block.Block) !f64 {
+ d.signalled += 1;
+ const cb = (try q.dspark_block.hold(d, blocks, .{ .event = d.event, .value = d.signalled })).?;
+ defer cb.release();
+ cb.wait();
+ if (cb.failure() != null) return error.GpuFailed;
+ return (cb.gpuEnd() - cb.gpuStart()) * 1000;
+}
+
+fn absorb(d: *q.dspark.Drafter, piece: q.dspark_block.Piece) !void {
+ const run = q.dspark_block.begin(d);
+ try q.dspark_block.absorbPass(d, run.e, &.{piece});
+ try q.dspark_block.end(run.cb, run.e);
+}
+
+/// Served rounds through the drafter's interface: after four warm rounds, no host allocation and one hold pass a round.
+fn steady(gpa: std.mem.Allocator, d: *q.dspark.Drafter, taps: mtl.Buffer, ids: []const u32, ctx: usize) !void {
+ const lanes = tf.lanes;
+ var s = try lanes.Stream.init(gpa, .{ .id = "steady", .prompt = ids[0 .. ctx + 1], .max_new = 160 });
+ defer s.deinit(gpa);
+ const dr = d.drafter();
+ try dr.open(&s);
+ defer dr.release(&s);
+ const width: u32 = @intCast(d.tapBytes());
+ const feats = struct {
+ fn at(buf: mtl.Buffer, row: usize, rows: usize, w: u32) lanes.backend.Features {
+ return .{ .buffer = @intFromPtr(buf.id), .offset = row * w, .rows = @intCast(rows), .row_bytes = w, .space = .device, .dtype = .bf16 };
+ }
+ };
+ try dr.absorb(&.{.{ .stream = &s, .features = feats.at(taps, 0, ctx, width), .start = 0, .follow = &.{} }});
+ var counted = std.testing.FailingAllocator.init(gpa, .{});
+ var anchor: u64 = ctx;
+ const rounds = 24;
+ const warm = 4;
+ var holds: u64 = 0;
+ for (0..rounds) |r| {
+ if (r == warm) {
+ d.gpa = counted.allocator();
+ holds = d.signalled;
+ }
+ try dr.hold(&.{.{ .stream = &s, .pending = ids[@intCast(anchor % ctx)], .position = anchor + 1, .depth = 3 }});
+ var chances: [3]f64 = undefined;
+ _ = try dr.chances(&s, &chances);
+ var drafts: [3]u32 = undefined;
+ try dr.held(&.{&s}, &.{&drafts});
+ const rows = r % 3 + 1;
+ const at: usize = @intCast(anchor % (ctx - 3));
+ try dr.absorb(&.{.{ .stream = &s, .features = feats.at(taps, at, rows, width), .start = anchor, .follow = &.{}, .prompt = false }});
+ anchor += rows;
+ }
+ d.gpa = gpa;
+ const n = rounds - warm;
+ std.debug.print("steady rounds: {d} rounds of hold, chances, held and a verified absorb: {d} host allocations, {d} resizes, {d} unwaited holds\n", .{ n, counted.allocations, counted.resize_index, d.signalled - holds });
+ if (counted.allocations != 0 or counted.resize_index != 0 or d.signalled - holds != n) return error.SteadyRoundsAllocate;
+}
+
+pub fn main(init: std.process.Init) !void {
+ const gpa = init.gpa;
+ const args = try init.minimal.args.toSlice(init.arena.allocator());
+ if (args.len < 4 or args.len > 6) return error.ExpectedModelDrafterPromptLengthRowsBits;
+ const bits: u8 = if (args.len == 6) try std.fmt.parseInt(u8, args[5], 10) else 0;
+ const length = try std.fmt.parseInt(usize, args[3], 10);
+ const pool = mtl.objc.Pool.push();
+ defer pool.pop();
+ const m = try q.Model.load(gpa, init.io, args[1]);
+ defer m.deinit();
+ const d = try q.dspark.Drafter.init(gpa, init.io, m, args[2], .{ .capacity = length + 64, .bits = bits });
+ defer d.deinit();
+ const cfg = d.config;
+ // the compared block's rows (a hold of depth d runs a d-row block)
+ const block: u32 = if (args.len >= 5) try std.fmt.parseInt(u32, args[4], 10) else cfg.block;
+ if (block == 0 or block > cfg.block) return error.BadBlockRows;
+ var scratch = try st.Scratch.init(gpa, m.device, &m.config.attention, length);
+ defer scratch.deinit();
+ const width = cfg.tap_count * c.hidden;
+ var taps: st.Taps = .{ .count = cfg.tap_count, .buf = try scratch.memory.alloc(length * width * 2) };
+ @memcpy(taps.layers[0..cfg.tap_count], cfg.taps[0..cfg.tap_count]);
+ scratch.taps = taps;
+ var cache = try st.Cache.init(gpa, m.device, &m.config.attention, length + 16);
+ defer cache.deinit();
+ const ids = try gpa.alloc(u32, length);
+ defer gpa.free(ids);
+ for (ids, 0..) |*id, i| id.* = @intCast((i * 7919 + 13) % 120000);
+ try q.forward.run(m, &scratch, &.{.{ .cache = &cache, .rows = length }}, ids, false, .last);
+ // the context: rows [0, length - 1); the pending token is the last id, at position length - 1
+ const ctx = length - 1;
+ const lane = try d.newLane(length + 32);
+ defer d.dropLane(lane);
+ var at: usize = 0;
+ while (at < ctx) : (at += q.dspark.absorb_rows) {
+ const rows = @min(q.dspark.absorb_rows, ctx - at);
+ try absorb(d, .{ .lane = lane, .src = .{ .buf = taps.buf, .off = at * width * 2 }, .rows = rows, .p0 = at });
+ }
+ lane.end = ctx;
+ _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null);
+ var held: [16]u32 = undefined;
+ @memcpy(held[0..block], lane.held[0..block]);
+ var conf: [16]f32 = undefined;
+ @memcpy(conf[0..block], lane.conf[0..block]);
+ const gpu = held[0..block];
+ for ([_]usize{ 1, 4, 16, 128, q.dspark.absorb_rows }) |rows| {
+ if (rows > ctx) continue;
+ var best: f64 = std.math.floatMax(f64);
+ for (0..5) |_| {
+ const t0 = std.Io.Clock.awake.now(init.io);
+ try absorb(d, .{ .lane = lane, .src = .{ .buf = taps.buf, .off = (ctx - rows) * width * 2 }, .rows = rows, .p0 = ctx - rows });
+ best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6);
+ }
+ std.debug.print("absorb of {d} rows: {d:.2} ms (best of 5)\n", .{ rows, best });
+ }
+ const depths = [_]u32{ 1, 3, cfg.block };
+ for (depths) |depth| {
+ var best: f64 = std.math.floatMax(f64);
+ for (0..5) |_| {
+ const t0 = std.Io.Clock.awake.now(init.io);
+ _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = depth }}, null);
+ best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6);
+ }
+ std.debug.print("hold at depth {d}: {d:.2} ms (best of 5)\n", .{ depth, best });
+ }
+ // a shared round's hold: S streams' blocks of d rows in one pass (S x d at most pass_rows), each on this context
+ var lanes_: [16]*q.dspark.Lane = undefined;
+ lanes_[0] = lane;
+ var made: usize = 1;
+ defer for (lanes_[1..made]) |l| d.dropLane(l);
+ while (made < lanes_.len) : (made += 1) {
+ const l = try d.newLane(length + 32);
+ lanes_[made] = l;
+ var from: usize = 0;
+ while (from < ctx) : (from += q.dspark.absorb_rows) {
+ const rows = @min(q.dspark.absorb_rows, ctx - from);
+ try absorb(d, .{ .lane = l, .src = .{ .buf = taps.buf, .off = from * width * 2 }, .rows = rows, .p0 = from });
+ }
+ l.end = ctx;
+ }
+ for ([_]usize{ 2, 4, 8, 16 }) |streams| for ([_]u32{ 1, 2, 3, 4, 8 }) |depth| {
+ if (streams * depth > q.dspark.pass_rows) continue;
+ var blocks: [16]q.dspark_block.Block = undefined;
+ for (blocks[0..streams], lanes_[0..streams]) |*b, l| b.* = .{ .lane = l, .pending = ids[ctx], .anchor = ctx, .depth = depth };
+ var best: f64 = std.math.floatMax(f64);
+ var gpu_best: f64 = std.math.floatMax(f64);
+ for (0..5) |_| {
+ const t0 = std.Io.Clock.awake.now(init.io);
+ _ = try q.dspark_block.hold(d, blocks[0..streams], null);
+ best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6);
+ gpu_best = @min(gpu_best, try gpuHold(d, blocks[0..streams]));
+ }
+ std.debug.print("hold of {d} streams at depth {d}: {d:.2} ms, GPU {d:.2} ms (best of 5)\n", .{ streams, depth, best, gpu_best });
+ };
+ try steady(gpa, d, taps.buf, ids, ctx);
+ _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null);
+ // the host reference, fp32 throughout
+ var r: Ref = .{ .gpa = gpa, .check = ck.Checkpoint.init(gpa), .cfg = cfg };
+ defer r.check.deinit();
+ const path = try std.fs.path.joinZ(gpa, &.{ args[2], "model.safetensors" });
+ defer gpa.free(path);
+ try r.check.addFile(m.device, path, "");
+ var arena = std.heap.ArenaAllocator.init(gpa);
+ defer arena.deinit();
+ r.gpa = arena.allocator();
+ const a = r.gpa;
+ const n = block;
+ const tap_rows = taps.buf.slice(u16, length * width);
+ const feats = try a.alloc(f32, ctx * width);
+ for (feats, tap_rows[0 .. ctx * width]) |*f, v| f.* = quant.toF32(v);
+ const hctx = try a.alloc(f32, ctx * c.hidden);
+ try matmul(gpa, try r.get("fc.weight"), feats, hctx, c.hidden, width, ctx);
+ const hn = try r.get("hidden_norm.weight");
+ for (0..ctx) |j| rms(hctx[j * c.hidden ..][0..c.hidden], hn);
+ const h = try a.alloc(f32, n * c.hidden);
+ for (0..n) |i| embedRow(m, if (i == 0) ids[ctx] else cfg.mask, h[i * c.hidden ..][0..c.hidden]);
+ const keys = ctx + n;
+ var name: [96]u8 = undefined;
+ const x = try a.alloc(f32, n * c.hidden);
+ for (0..cfg.layers) |li| {
+ const kmat = try r.layer(&name, li, "self_attn.k_proj");
+ const vmat = try r.layer(&name, li, "self_attn.v_proj");
+ const qn = try r.layer(&name, li, "self_attn.q_norm");
+ const kn = try r.layer(&name, li, "self_attn.k_norm");
+ const kk = try a.alloc(f32, keys * kw);
+ const vv = try a.alloc(f32, keys * kw);
+ try matmul(gpa, kmat, hctx, kk, kw, c.hidden, ctx);
+ try matmul(gpa, vmat, hctx, vv, kw, c.hidden, ctx);
+ @memcpy(x, h);
+ for (0..n) |i| rms(x[i * c.hidden ..][0..c.hidden], try r.layer(&name, li, "input_layernorm"));
+ const qq = try a.alloc(f32, n * qw);
+ try matmul(gpa, try r.layer(&name, li, "self_attn.q_proj"), x, qq, qw, c.hidden, n);
+ try matmul(gpa, kmat, x, kk[ctx * kw ..], kw, c.hidden, n);
+ try matmul(gpa, vmat, x, vv[ctx * kw ..], kw, c.hidden, n);
+ for (0..keys) |j| for (0..c.kv_heads) |kh| {
+ const row = kk[j * kw + kh * hd ..][0..hd];
+ rms(row, kn);
+ rope(row, j, cfg.theta);
+ };
+ for (0..n) |i| for (0..c.query_heads) |qh| {
+ const row = qq[i * qw + qh * hd ..][0..hd];
+ rms(row, qn);
+ rope(row, ctx + i, cfg.theta);
+ };
+ const o = try a.alloc(f32, n * qw);
+ const sc = try a.alloc(f32, keys);
+ for (0..n) |i| for (0..c.query_heads) |qh| {
+ const kh = qh / (c.query_heads / c.kv_heads);
+ var top: f32 = -std.math.inf(f32);
+ for (0..keys) |j| {
+ var s: f32 = 0;
+ for (0..hd) |t| s += qq[i * qw + qh * hd + t] * kk[j * kw + kh * hd + t];
+ sc[j] = s / @sqrt(@as(f32, hd));
+ top = @max(top, sc[j]);
+ }
+ var sum: f32 = 0;
+ for (sc) |*s| {
+ s.* = @exp(s.* - top);
+ sum += s.*;
+ }
+ for (0..hd) |t| {
+ var acc: f32 = 0;
+ for (0..keys) |j| acc += sc[j] * vv[j * kw + kh * hd + t];
+ o[i * qw + qh * hd + t] = acc / sum;
+ }
+ };
+ const y = try a.alloc(f32, n * c.hidden);
+ try matmul(gpa, try r.layer(&name, li, "self_attn.o_proj"), o, y, c.hidden, qw, n);
+ for (h, y) |*v, w| v.* += w;
+ @memcpy(x, h);
+ for (0..n) |i| rms(x[i * c.hidden ..][0..c.hidden], try r.layer(&name, li, "post_attention_layernorm"));
+ const gate = try a.alloc(f32, n * cfg.intermediate);
+ const up = try a.alloc(f32, n * cfg.intermediate);
+ try matmul(gpa, try r.layer(&name, li, "mlp.gate_proj"), x, gate, cfg.intermediate, c.hidden, n);
+ try matmul(gpa, try r.layer(&name, li, "mlp.up_proj"), x, up, cfg.intermediate, c.hidden, n);
+ for (gate, up) |*g, u| g.* = g.* / (1 + @exp(-g.*)) * u;
+ try matmul(gpa, try r.layer(&name, li, "mlp.down_proj"), gate, y, c.hidden, cfg.intermediate, n);
+ for (h, y) |*v, w| v.* += w;
+ }
+ const fin = try r.get("norm.weight");
+ for (0..n) |i| rms(h[i * c.hidden ..][0..c.hidden], fin);
+ const head = try a.alloc(f32, c.vocab * c.hidden);
+ for (0..c.vocab) |v| embedRow(m, @intCast(v), head[v * c.hidden ..][0..c.hidden]);
+ const base = try a.alloc(f32, n * c.vocab);
+ try matmul(gpa, head, h, base, c.vocab, c.hidden, n);
+ const w1 = try r.get("markov_head.markov_w1.weight");
+ const w2 = try r.get("markov_head.markov_w2.weight");
+ const bias = try a.alloc(f32, c.vocab);
+ const proj = try r.get("confidence_head.proj.weight");
+ const proj_bias = (try r.get("confidence_head.proj.bias"))[0];
+ var conf_worst: f32 = 0;
+ var prev = ids[ctx];
+ var same: usize = 0;
+ var worst: f32 = 0;
+ for (0..n) |i| {
+ try matmul(gpa, w2, w1[@as(usize, prev) * cfg.rank ..][0..cfg.rank], bias, c.vocab, cfg.rank, 1);
+ const row = base[i * c.vocab ..][0..c.vocab];
+ var best: usize = 0;
+ var second: usize = 1;
+ for (row, bias, 0..) |b, m2, v| {
+ const s = b + m2;
+ if (s > row[best] + bias[best]) {
+ second = best;
+ best = v;
+ } else if (v != best and s > row[second] + bias[second]) second = v;
+ }
+ const margin = (row[best] + bias[best]) - (row[second] + bias[second]);
+ if (i == 0) { // the GPU keeps the base logits (the Markov step adds its bias on the way to the pick)
+ const zero = try a.alloc(f32, c.vocab);
+ @memset(zero, 0);
+ compare("row 0 base logits", d.s.logits.slice(u16, c.vocab), row, zero);
+ }
+ const ok = gpu[i] == best;
+ same += @intFromBool(ok);
+ std.debug.print("step {d}: reference {d} (margin {d:.3}, second {d}), GPU {d} {s}\n", .{ i, best, margin, second, gpu[i], if (ok) "equal" else "DIFFERENT" });
+ if (!ok) worst = @max(worst, margin);
+ // the confidence head: [final norm row i, W1 of the token before draft i] . proj + bias, through a sigmoid
+ var logit: f32 = proj_bias;
+ for (h[i * c.hidden ..][0..c.hidden], proj[0..c.hidden]) |x0, w0| logit += x0 * w0;
+ for (w1[@as(usize, prev) * cfg.rank ..][0..cfg.rank], proj[c.hidden..]) |x0, w0| logit += x0 * w0;
+ const want = 1 / (1 + @exp(-logit));
+ conf_worst = @max(conf_worst, @abs(want - conf[i]));
+ std.debug.print("step {d}: confidence reference {d:.4}, GPU {d:.4}\n", .{ i, want, conf[i] });
+ prev = gpu[i]; // teacher-forced: step i + 1 follows the GPU's pick, so each step is compared alone
+ }
+ std.debug.print("the reference's largest margin over a different GPU pick: {d:.3}\n", .{worst});
+ std.debug.print("the confidences' worst difference from the fp32 reference: {d:.5}\n", .{conf_worst});
+ std.debug.print("{d} of {d} drafts equal the fp32 reference's picks on the same chain ({d}-token context)\n", .{ same, n, ctx });
+}