diff --git a/README.md b/README.md index 467380fd36..04cb43a948 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ Conversions are explicitly marked: their upstream weights need the linked conver | GLM-5.3-Flash, FP4-Local | `tensorfold pull nvidia/GLM-5.3-Flash-FP4-Local@76348ddb2be2e35e39ecfc00e75dc07703f4f3d5` | Two GB10s at TP=2, or three at TP=3 over a ring of direct links: @Bizuayeu loaded it as stored and served it at both sizes. 165.3 GiB of weights, 13% less than NVFP4's 190.4 GiB. | [Native CUDA](docs/recipes/glm-5.3-flash-cuda.md), in 1.1.5 (#1050). NVIDIA's mixed-precision checkpoint: NVFP4 routed experts with compressed block scales, MXFP8 and 128×128-block FP8 for most of the rest. As close to BF16 as NVFP4 at its smaller size: at TP=2 its NLL is 1.11% above BF16's against NVFP4's 1.37% ([contributed receipt](docs/receipts/glm-fp4local-80517fc3/README.md) on #1050's head, not rerun on main), a difference the per-document intervals in [XSFP4's card](https://huggingface.co/Bizuayeu/GLM-5.3-Flash-XSFP4#measured) do not resolve. | | GLM-5.3-Flash, XSFP4 | `tensorfold pull Bizuayeu/GLM-5.3-Flash-XSFP4@88736f916f8939607fde8a4b881dcae749f7551d` | Two GB10s at TP=2, or three at TP=3 over a ring of direct links: on main `8086c205c` @Bizuayeu loaded it in 370.7 s at TP=2 (rank 0's model 96.97 GiB at a 524,288-token window) and in 313.8 s at TP=3 (63.60 GiB at 1,048,576), and both served. 165.7 GiB of weights, 0.4 GiB more than FP4-Local's. | [Native CUDA](docs/recipes/glm-5.3-flash-cuda.md), in 1.1.5 through FP4-Local's path. FP4-Local with its routed experts' NVFP4 blocks fitted again from the BF16 weights, a smoothing folded into them, so it runs on the same kernels. Closer to BF16 than FP4-Local and NVFP4 at TP=2 by all three measures in [its card](https://huggingface.co/Bizuayeu/GLM-5.3-Flash-XSFP4#measured), on #1050's code: top-1 agreement 93.26%, NLL 0.85% above and a KL bound of 0.0431, against FP4-Local's 92.82%, 1.11% and 0.0510. | | GLM-5.3, EXLR8 `home` | `tensorfold pull kindlingai/glm-5.3-exlr8-k2-k3-k4@f459f5d0272a529403dd9b3fd32f771f4250a0d4 --exlr8 home --node 0/2 --dir ~/models/glm-5.3-exlr8-home` | Two M5 Ultra Macs, 256 GB each; about 160 GB resident per Mac at 8k context / four streams. | [Two-Mac recipe](docs/glm-5.3-exlr8.md). Use `--node 1/2` on the other Mac; Thunderbolt 5 and MCDMA required. No CUDA backend. | -| LFM2.5-24B-A2B, MLX 4 / 6 / 8-bit | `tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-4bit@667408d494bde191c2cb9dd4f4549827816b9275`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-6bit@1deec53eb49571e21cdcbf82fd5839248f07ed88`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-8bit@67ab417dd02d0c9545802a726df668620eae049d` | One Mac; M4 Pro checked. | [Native recipe](docs/recipes/lfm2.5-24b-a2b.md), [release history](CHANGELOG.md). | +| LFM2.5-24B-A2B, MLX 4 / 6 / 8-bit | `tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-4bit@667408d494bde191c2cb9dd4f4549827816b9275`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-6bit@1deec53eb49571e21cdcbf82fd5839248f07ed88`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-8bit@67ab417dd02d0c9545802a726df668620eae049d` | One Mac; M4 Pro checked. | [Native recipe](docs/recipes/lfm2.5-24b-a2b.md), [release history](CHANGELOG.md). LiquidAI's DSpark drafter with `--drafter`. | | Qwen3.6-35B-A3B, MLX with MTP | `tensorfold pull TensorFold/Qwen3.6-35B-A3B-MLX-4bit-MTP@81169a9bc511a27c1b4eedb77a2cd98ced431847` | One GB10; 20.9 GB checkpoint. | [Qwen3.6](docs/recipes/qwen3.6-moe.md). Native CUDA; the recipe's Metal path belongs to Python. | | Gemma 4 31B, MLX 4-bit | `tensorfold pull mlx-community/gemma-4-31B-it-4bit@696d436c404745a59f30e4939a658162b0a9e57f`
`tensorfold pull z-lab/gemma-4-31B-it-DFlash@eabd648301ce28583cc14757912e5e0f84e152e1` | One M5 Mac; M5 Ultra checked. M1-M4 refused. | [Native 31B](docs/recipes/gemma-4-native.md). Name the drafter with `--drafter`; images with `--vision`. | | Gemma 4 12B, converted MLX 4-bit | `hf download google/gemma-4-12B-it --revision 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 --local-dir ~/models/gemma-4-12b-hf`
`hf download google/gemma-4-12B-it-assistant --revision 46d4c6f13f0ac0ad827b915669b8df9b81c64c51 --local-dir ~/models/gemma-4-12b-assistant` | One M5 Mac; converted text weights 6.3 GB, image weights 100 MB, plus assistant/caches. M1-M4 refused. | [Convert and serve](docs/recipes/gemma-4-12b-native.md). Upstream download needs `hf`; conversion needs Python, mlx and mlx-lm. Serving is native. | @@ -290,7 +290,7 @@ The binary's `capabilities --json` response lists its supported flags and platfo | `--loop-guard-stop` | As `--loop-guard`, but end the reply where the guard fires: the reasoning so far, no answer or tool calls, `finish_reason` `content_filter` and a `reasoning_loop` error ([API](docs/api.md)). | | `--no-drafts` | Produce the plain reference through the same engine. | | `--drafter DRAFTER` | An external drafter, as a directory or a repo id in the Hugging Face cache, for models that take one: Qwen3.8-27B uses DFlash2 and Gemma 4 31B the drafter named in its recipe. Without one they decode plain. | -| `--drafter-bits 0\|4` | Drafter weight precision where the model offers a choice: 4 bits (the default), or 0 to keep bf16. Qwen3.8-27B on CUDA takes 4 only. | +| `--drafter-bits 0\|4\|8` | Drafter weight precision where the model offers a choice: 4 bits (the default), or 0 to keep bf16. Qwen3.8-27B on CUDA takes 4 only. LFM2.5-24B-A2B's DSpark drafter runs bf16 as released (0, the default when not given) or 8-bit codes quantized at load (8); it has no 4-bit mode. | | `--record DIR` | Write each stream's committed tokens, positions and tapped rows under DIR, for drafter training. CUDA Nemotron only; a single stream runs through the lane rounds, and recording stops below 25 GiB free disk. | | `--keep-warm SECONDS` | Keep the Metal GPU active while idle for this long after a request, default 900; zero disables it. | | `--cfg-gamma GAMMA` | Metal Drama speech: semantic guidance strength, finite 1–5; 0/default disables it. Only reference-conditioned requests are guided. Enabled replies report `X-Cfg-Gamma`; codec heads stay conditional. | diff --git a/docs/kernel-contract.md b/docs/kernel-contract.md index 1ed43ac146..b04aa6996b 100644 --- a/docs/kernel-contract.md +++ b/docs/kernel-contract.md @@ -135,8 +135,10 @@ tile_qmm's pipelines and GLM-5.3-Flash's hc_expand_10. LFM2.5-24B-A2B's shared rounds: the greedy pick (`lfm2_argmax`) and the conv and attention kernels that run every stream's rows in one dispatch, each bit-equal to the per-stream kernels it replaces. Four controls edit the source: a tie to the highest id, a nonfinite logit not flagged, and every stream reading or writing the first stream's state or cache. -Their launches are in `lfm2_records.zig` with the M1 and M2 limits from `--offline`; `lfm2_attn_segs` reserves its 128 -threads in its source, as Metal cannot build it for M1 or M2 at another count. +The DSpark drafter's tap copy, Markov scan and pick, draft ids, block attention and confidence logit are there too, with seven more controls. +Its bf16 linears (few rows, 64x64 tiles) and the Markov scan over a bf16 W2 have four and two more. +Their launches are in `lfm2_records.zig` with the M1 and M2 limits from `--offline`; `lfm2_attn_segs` and +`lfm2_attn_block` reserve their 128 threads in their source, as Metal cannot build them for M1 or M2 at another count. Metal also checks Nemotron's routed experts on the engine's route tables, one row to 128: `tf_xup_w3` and `tf_xdown_w3` against float64 at tf_load16's bf16 x sums (up's relu2 as an interval over its two bf16 roundings), and the rows2, rows4 and six geometry pipelines of each projection bit-equal to them (two controls: member rows read from one row, and diff --git a/docs/receipts/lfm2-dspark-review/README.md b/docs/receipts/lfm2-dspark-review/README.md new file mode 100644 index 0000000000..4a2a6f409a --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/README.md @@ -0,0 +1,273 @@ +# DSpark review fixes on current main + +Source `14565fa73`, rebased onto main `d0c9dc9fb`, tested on 2026-10-28. +Apple M4 Pro, 20 GPU cores, 48 GiB RAM, macOS 26.5.1, Zig 0.17.0, +Apple Metal compiler 32023.883. Builds target `apple_m1`. + +The shared and lone guards from the rejected PRs are absent. The lane core retains +confidence/grant pricing, hold-cost pricing and `copy_first`. Hold ordering reuses +one bounded allocation instead of allocating on every round. The initialization +cleanup releases Metal buffers before their owning list; injected scratch and warm +allocation failures exercise that path. Config admission refuses rank above 256 and +nonfinite f32 RoPE theta. The quantizer test widens its size calculation before +multiplication, and `--drafter` refuses a target restricted to one-row verification. + +## Checks + +- `zig build test test-kernel-contract test-golden fuzz-cortheon native dev-tools -Dcpu=apple_m1 --summary all`: + 749/749 steps succeeded; 8,414 tests passed, 46 skipped, 0 failed. +- `zig build -Dcpu=apple_m1 --summary all`: 239/239 steps succeeded. +- `zig build test-lfm2 test-lfm2-dspark -Dcpu=apple_m1 --summary all`: + 111/111 tests passed. The final scratch/warm failure test also passed in a subsequent + focused run: 51/51 DSpark tests. +- `zig build test-lfm2-dspark -Dcpu=apple_m1 -Doptimize=debug --summary all`: + 51/51 passed, including both injected allocation failures. +- Debug `zig test` on `dspark_config.zig`: 4/4 passed; on `dspark_quant.zig`: 2/2 passed. + The original quantizer test from `2e5966f68` aborted with integer overflow in Debug. +- `zig fmt --check zig` and `git diff --check`: passed. +- Lean: 428 findings on main, 428 on this source, 0 new. +- `tf-kernel-contract --only lfm2`: 11 kernels passed gates 1-4, 0 failures, + 0 known findings; all 17 planted controls red, 0 missed; 0 limit problems. +- `tf-kernel-contract --only tile_qmm`: 6 kernels passed gates 1-4, 0 failures, + 0 known findings; 0 limit problems. +- `tf-lfm2-dspark MODEL DRAFTER 256`: 9/9 drafts equal the independent fp32 + reference's picks; maximum confidence difference 0.00106; 20 steady drafter + rounds had zero host allocations and resizes; exit 0. +- Completed `tf-lfm2-exact`: 72/72 checks on both M4 Pro and M5 Ultra, zero + failures or skipped checks. This supersedes the earlier interrupted run. + +## Served equality + +Target `LiquidAI/LFM2.5-24B-A2B-MLX-4bit` at +`667408d494bde191c2cb9dd4f4549827816b9275`, with the local released DSpark weights. +The drafter safetensors SHA-256 is recorded in the accompanying JSON. + +For both default bf16 and explicit `--drafter-bits 8`, serve with `--parallel 16 +--context 4096 --prompt-cache-gib 0 --keep-warm 0`. Run: + +```sh +python3 tools/bench_concurrent.py http://127.0.0.1:18194 review \ + --tokens 64 --reps 1 --temperatures 0 --levels 1,4,8,16 --alone --serial +python3 tools/bench_concurrent.py http://127.0.0.1:18194 review \ + --tokens 64 --reps 1 --temperatures 0 --levels 1,4,8,16 --alone --serial \ + --prompts tools/bench_prompts.json +``` + +Each precision mode passed all 12 cells: eight same-prompt cells (code and chat) +and four distinct-prompt cells. Concurrent replies equal solo replies and drafted +solo replies equal `"draft": false` token hashes. Zero unequal replies and zero +failed requests. [Equality evidence](equality-and-load.json) retains per-request +concurrent, solo and plain token hashes. + +These are short correctness checks, with one repetition, while builds were also +running. Their timings do not establish the decode performance gate. + +## Peak during load + +One model process at a time under a model lock. Both servers used the same target +and bf16 drafter, `--parallel 1 --context 4096 --prompt-cache-gib 0 --keep-warm 0`. +Read `/memory` after readiness and before sending any request. Its lifetime peak +agreed with `/usr/bin/time -l` at process exit. + +- Exact #1392 head `1bb6e5db2`: 16,036,097,352 bytes, 14.935 GiB. +- Corrected stack source `14565fa73`: 15,527,930,776 bytes, 14.462 GiB. + +The measured reduction during load is 508,166,576 bytes (484.6 MiB). This compares +the original pre-allocation head with the corrected stack rebased onto current +main; it is not an isolated comparison of #1393's commit on the same base. + +## Extended qualification on the published stack + +Source `2dbbc0e8f`, main baseline `d0c9dc9fb`. Additional device: Apple M5 Ultra, +256 GiB RAM, macOS 27.0, Metal compiler 32023.921.6, Zig 0.17.0. +[Extended evidence](extended-qualification.json) retains request hashes, counters, +per-round throughput and latency. Generated text and token arrays are omitted. +[Reproduction harness](qualification.py) supports `functional`, `long`, +`performance`, `performance qwen`, and `cold`; set `TF_QUAL_MAIN`, +`TF_QUAL_CANDIDATE`, `TF_QUAL_MODEL`, `TF_QUAL_DRAFTER`, and `TF_QUAL_OUT` +for the local installations. Run one model process at a time. + +Both devices passed real two-turn resumed-versus-fresh chat equality (3,517 cached +prompt tokens), 29 seeded sampled equality cases across 1/4/8/16 streams, and +cancellation/replacement stress with Metal API validation enabled. M4 completed +143 requests with 84 cancellations at six slots and 252 with 134 at sixteen; +M5 completed 453 with 247 and 852 with 459 respectively. Zero wrong replies or +request errors. Server shutdowns were graceful. + +M5 plain/drafted serving hashes agree at 4,096, 16,384, 65,536 and 130,816 prompt +tokens. Independent fp32 drafter checks agree on all nine picks at histories +65,535 and 130,815; maximum confidence differences 0.00532 and 0.00351. +Twenty steady rounds had zero host allocations/resizes. M5 also passes the same +11 LFM2 and six tile_qmm contracts, with all 17 LFM2 planted controls red. + +### Performance remains an open issue + +Three alternating rounds per arm, 256 generated tokens, 1/4/8/16 streams, +32-token warmups and uncached prompts. Aggregate decode rate uses the span from +the earliest first token to the latest last token. The `chat` fixture label in +the JSON means prose submitted to raw completions, not a templated chat request. +All cross-arm output hashes agree. These results do **not** clear a no-loss gate. + +On M5, drafted versus main-plain median decode changes for code at 1/4/8/16 +streams are +19.44%, -0.96%, -2.20%, -4.23%; for prose they are +35.89%, +-0.72%, -1.25%, -1.17%. Candidate-plain closely follows main-plain. +On M4, changes are approximately -6.0%, -13.9%, -5.6%, +0.7% for code and +-6.4%, -13.4%, -7.8%, -8.0% for prose. M4 also shows variation between plain +arms, so these measurements do not isolate every source of the loss. + +M5 cold prefill medians at 2k/8k/32k/64k tokens are 0.1527/0.6027/2.9244/7.3276 +seconds on main-plain versus 0.1634/0.6327/3.0466/7.5652 with the drafter. +Prompt counts were checked and cached-token counts were zero. + +Other-family Qwen3.8-27B plus DFlash2 was tested at one and four streams with +three alternating rounds on both devices. M4 results vary with each executable's +independently measured persistent cost record: fresh records give code changes +of -7.13%/+3.71% and prose -6.47%/+0.83%. The original records and fresh-record +runs are both retained. No timings were copied between executables. M5 changes are +0.20%/-0.47% for code and +0.05%/+0.70% for prose. +No blanket other-family performance pass is claimed. + +Two isolated, uncommitted experiments (a larger absorb staging buffer and +removing forced per-stream copy pricing) did not consistently remove the loss. +Neither experiment is part of this PR. Further pricing work requires its own +validation; the rejected shared/lone guards have not been restored. + +M1, M2, M3, CUDA and HIP were not rerun for this revision. Focused Metal checks +are not a full cross-family kernel-contract receipt. All nine GitHub checks passed on `2dbbc0e8f` before this receipt-only update. + +### Performance follow-up (`b4deee277`, M4 and M5) + +Candidate source `b4deee277` keeps context ingestion on the original bf16 row +kernel through 32 rows, and uses the existing tiled kernel for held blocks past +16 rows. It measures each admissible held depth and interpolates neighboring +measured prices. The hold-price unit test covers a nonlinear timing curve. +Qwen/DFlash2 dispatch and scheduling are unchanged; only LFM2 supplies this hold +price table. + +`performance-investigation.json` retains all 64 M4 comparison cells, their output +hashes, source hashes and actual arm order. Four-round median changes against +the published PR are +1.06%, +0.55%, effectively flat and +0.57% for code at +1/4/8/16 streams; prose changes are +0.27%, +0.75%, +0.41% and +0.42%. +The 43.99 tok/s lone-code outlier is retained. The final arm order repeats the +previous order, so this is not a perfectly balanced four-round experiment. +All reply hashes agree. Independent fp32 drafter picks agree in all nine cases. +Focused DSpark, shared-lane equality and native-option checks pass: 49 build +steps and 56 tests. Formatting, whitespace and harness syntax checks pass. +These small gains do not establish a no-loss result against main. + +An earlier candidate changed context projections as well as held blocks and +regressed M4 16-stream prose by 8.38%. Preserving context projections removes +that regression in this comparison, supporting context arithmetic as a cause. +A separate acceptance-cap scheduler experiment lost roughly 2–3% in some M5 +cells and is excluded. Faster individual holds alone did not guarantee faster +serving. The exact retained candidate also completed M5 validation, with source hashes +matching this tree and tested executable hashes retained in the JSON receipt. + + +M5 completed 72 cells in three Latin rotations of main without the external +DSpark drafter, the published PR and the candidate. Four-round M4 and three-round +M5 results use the same 256-token requests and aggregate decode metric. The main +arm still permits suffix copying; it is not the literal `"draft": false` oracle. +Candidate versus published median changes at 1/4/8/16 streams are -0.26%, +0.04%, ++0.31%, +0.58% for code and -0.04%, -0.02%, +0.06%, +0.72% for prose. +Against main, code is +21.87%, -0.90%, -2.22%, -3.51%; prose is +36.18%, -0.55%, +-1.19%, -0.36%. Wider concurrency therefore still does not clear a no-loss gate. + +Final M5 LFM2 contracts: 11 pass, zero failures, all 17 planted controls red, +zero limit problems. Independent FP32 drafter picks agree in all nine cases, +with zero host allocations/resizes in 20 steady rounds. Resumed/fresh equality, +29 seeded sampled cases, eight 8-bit cells across 1/4/8/16 streams, and long +contexts of 65,536 and 130,816 tokens (drafted/undrafted and four concurrent/solo) +all pass. Cancellation stress completes 423 requests with 234 cancellations at +six slots and 827 with 445 cancellations at sixteen slots, zero wrong replies +or request errors. All servers shut down gracefully. Hashes and counters are in +`performance-investigation.json`. This extends, rather than replaces, the +historical full forward/cache and qualification receipts above. + +After that qualification, main `dd0a9cc5e` was merged to resolve a new CLI +conflict. The resolution retains main's drafter-name reporting and this stack's +optional precision default. Native plus the same focused regression targets +pass: 66/66 build steps, 56/56 tests. Lean compares at 425 findings on both main +and the merged tree, zero new. Performance and served-GPU receipts above are +for `b4deee277` before this merge; they are not relabeled as exact merged-head +measurements. + +### Wide-batch calibration (`099455490`) + +LFM2 timed single-stream verification windows through 16 rows, but the core may +verify 32 shared rows. It therefore priced the extra rows by proportional +extrapolation. This revision measures the shared path with two individually +qualified windows, and anchors its measured increments to the existing 16-row +window cost. Per-stream qualification remains 16 rows. The granted-depth rule +also reuses its first copy-proposer decision: asking again can mutate the +proposer's silence state and change that same decision. A stateful regression +reproduces the duplicate-query bug. Qwen/DFlash2's policy is unchanged. + +`wide-calibration-candidate.json` retains the exact source and executable hashes, +checkpoint/tokenizer identities, rendered prompt IDs, all 128 cells, request +SSE timing spans, output hashes and counters. Four Latin rotations compare main +`dd0a9cc5e`, published head `68c7435b0`, candidate without the drafter, and +candidate with bf16 drafting. Every output hash agrees. The plain performance +arms keep suffix copying enabled; literal `"draft": false` is the separate +correctness oracle. + +Against the published PR, candidate drafted code changes at 1/4/8/16 streams are ++0.48%, **-1.93%**, -0.01%, **+10.96%**; prose changes are +0.05%, -0.42%, ++0.50%, **+8.86%**. At 16 streams it also beats main: 886.46 versus 830.13 tok/s +for code, and 862.91 versus 798.61 for prose. The task owner accepted the +four-stream tradeoff for the larger sixteen-stream gain. This is not a universal +no-loss claim. The candidate without the drafter also loses 2.04% on four-stream +prose versus main; all four arms are retained, including that result. + +M5 final correctness passes all 11 LFM2 contracts, with all 17 planted controls +red and zero limit problems; 9/9 independent FP32 drafter picks; 20 steady rounds +with zero host allocations/resizes; resumed/fresh equality with 3,517 reused +prompt tokens; 29 seeded sampled cases; eight 8-bit cells; and drafted/plain plus +four-concurrent/solo equality at 65,536 and 130,816 prompt tokens. Six-slot stress +completes 423 requests with 233 cancellations; sixteen-slot stress completes +845 with 455 cancellations. Zero wrong replies or request errors; graceful +shutdowns throughout. + +M4 passes the same 11 operator contracts and 17 controls, 9/9 independent drafter +picks (worst confidence difference 0.00106), and 20 allocation-free steady +rounds. Its served run stops at the resumed-cache prerequisite: the OS reports +memory pressure and explicitly disables keeping prompt states, so reused tokens +remain zero. Its later stress/sampled/8-bit phases did not run. No pressure guard +was bypassed, and no new M4 serving or throughput qualification is claimed. + +Local native/family/shared-lane checks pass 69/69 steps and 117/117 tests. The +default build passes 239/239 steps. The direct proposer regression, 14 granted +and 34 copy-policy tests also pass. M5 pre-request, context-4096, one-stream bf16 +load peaks are 15,709,645,744 bytes for published and 15,733,451,768 for candidate +(+22.7 MiB, one run each). Hot startup increases by roughly one second. Other +chips/backends and the full cross-family operator gate were not rerun. + +`wide-calibration-ablations.json` retains 765 M5 diagnostic cells and 35 interrupted, +contended M4 cells. Interrupted runs are marked; they are not qualification +passes. Larger staging, changed copy-rate policies, confidence fallbacks and +uniform late-hold budgeting were discarded rather than included in this patch. + +To reproduce the four-arm comparison, reserve the GPU exclusively and build +all three checkouts with `zig build native -Dcpu=apple_m1 -Dversion=1.1.8`, then run: + +```sh +TF_QUAL_MAIN=/path/to/main/zig-out/native/bin/tensorfold-native \ +TF_QUAL_PUBLISHED=/path/to/published/zig-out/native/bin/tensorfold-native \ +TF_QUAL_CANDIDATE=/path/to/candidate/zig-out/native/bin/tensorfold-native \ +TF_QUAL_CANDIDATE_ROOT=/path/to/candidate \ +TF_QUAL_DRAFTER=/path/to/LFM2.5-24B-A2B-DSpark \ +TF_QUAL_OUT=/tmp/tf-lfm2-wide-matrix \ +python3 docs/receipts/lfm2-dspark-review/wide_matrix.py +``` + +`TF_QUAL_MODEL` overrides the default target snapshot path. The reproducer uses +only Python's standard library and the sibling `qualification.py` helper. + +### Allocation follow-up + +Probability fallback now overwrites its existing output buffer when a provider declines. Grafted probabilities fill the draft prefix directly in the final output. Structured-output held tokens are read directly into the final row buffer before grafts. Probability multiplication order, copy rates, parent indexing and decoding policy are unchanged. + +For `n` fallback probabilities this saves `8*n` arena bytes. For `d` drafted prefix nodes it saves `8*d` arena bytes plus their copy. With `h` held tokens and `g` graft tokens, grammar row assembly saves `4*(2*h+g)` arena bytes and the held/joined intermediate copies. These are per affected call, not reductions in model weights or a measured serving peak. + +The bounded-workspace regression permits only the final probability buffer, denies allocator growth, and covers both declined providers and grafted prefixes. The previous implementation fails it with OutOfMemory; the refactor passes. A grammar regression covers held-token and graft order, including replacing a rejected graft with grammar-generated tokens. The shared-lane/native host checks pass 57/57 steps, 341 tests passed and three skipped. The Metal draft-source fixture now initializes the shared timing count introduced by the calibration change; otherwise its deliberately undefined backend could crash. + +The M5 native build passes. An alternating before/after serving comparison at 1/4/8/16 streams is queued behind another qualification's GPU lock. No extra throughput improvement or universal no-regression result is claimed for this allocation-only follow-up yet. diff --git a/docs/receipts/lfm2-dspark-review/equality-and-load.json b/docs/receipts/lfm2-dspark-review/equality-and-load.json new file mode 100644 index 0000000000..f66ec3b0d4 --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/equality-and-load.json @@ -0,0 +1,4517 @@ +{ + "source_commit": "14565fa73", + "base_commit": "d0c9dc9fb", + "target_revision": "667408d494bde191c2cb9dd4f4549827816b9275", + "drafter_weights_sha256": "7d06a78556b0b1f386134508f09b00a3a7e0e5e0ec2cc73119229f4ef7c9b299", + "tokens": 64, + "seed": 1234, + "repetitions": 1, + "ignore_eos": true, + "prompt_cache_gib": 0, + "load_comparison": { + "before-alloc-bf16-load": { + "parallel": 1, + "context": 4096, + "mode": "bf16", + "memory": { + "source": "darwin_rusage_info_v4", + "physical_footprint_bytes": 15464950680, + "lifetime_peak_physical_footprint_bytes": 16036097352 + } + }, + "candidate-bf16-load": { + "parallel": 1, + "context": 4096, + "mode": "bf16", + "memory": { + "source": "darwin_rusage_info_v4", + "physical_footprint_bytes": 15504747440, + "lifetime_peak_physical_footprint_bytes": 15527930776 + } + } + }, + "cells": [ + { + "mode": "bf16", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + } + } + ] + } + ] +} diff --git a/docs/receipts/lfm2-dspark-review/extended-qualification.json b/docs/receipts/lfm2-dspark-review/extended-qualification.json new file mode 100644 index 0000000000..db8d93df99 --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/extended-qualification.json @@ -0,0 +1,53517 @@ +{ + "source": "2dbbc0e8f5bbadf1a33351769439026f318ddb8f", + "main": "d0c9dc9fb", + "date": "2026-10-28", + "devices": { + "M4 Pro": { + "functional": { + "resumed": { + "usage": { + "prompt_tokens": 3605, + "completion_tokens": 64, + "total_tokens": 3669, + "prompt_tokens_details": { + "cached_tokens": 3517 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.39393036340272, + "seconds": 0.592139042, + "prefill_seconds": 0.181146458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.181769458, + "sampling": "greedy", + "drafts": true, + "token_sha": "a064a0a2711c", + "token_sha256": "a064a0a2711ce3e5053bd664c3540fc534b5f6044a0aa69b70bce596081d429e", + "min_rows": 2 + }, + "speculative": { + "rounds": 31, + "drafted": 55, + "accepted": 33, + "acceptance_rate": 0.6, + "tokens_per_round": 2.064516129032258 + }, + "wall": 0.7745752497576177 + }, + "resume_request": { + "messages": [ + { + "role": "user" + }, + { + "role": "assistant" + }, + { + "role": "user" + } + ], + "max_tokens": 64 + }, + "stress_reference_hashes": [ + { + "prompt": "quicksort-raw", + "sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0" + }, + { + "prompt": "rust-reverse-words", + "sha256": "d9499a1af105f9d0477f4fda4f2afde556c782008fe552d4a7cd0ebf71e80101" + }, + { + "prompt": "multiply-17-23", + "sha256": "3ac253c3ae100e4ca9328988d95e9324ab6de6f85787f191d13c718bfae1f4b4" + }, + { + "prompt": "sleep-tips", + "sha256": "076c7d4d0802aa0ee09b60f396deab8a6cfc5683e2f17b6b90f587b410d20ba5" + }, + { + "prompt": "js-debounce-raw", + "sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458" + }, + { + "prompt": "linear-equation", + "sha256": "9277e1d182a9cfc39ec6df2bb5f33c9acb4ca3db6922f5113769afeb3c2ee38b" + }, + { + "prompt": "fishing-village", + "sha256": "522136a9089e45996543679366846beba531361185bdc02baa0f336231b8848e" + }, + { + "prompt": "sql-left-join", + "sha256": "45afcf2bc8f7bdc6b1a1ae93f7a06c53e7a98db5e4d9f283753159d8e2707a74" + } + ], + "stress": [ + { + "slots": 6, + "seconds": 60, + "completed": 143, + "cancelled": 84, + "wrong": 0, + "errors": [] + }, + { + "slots": 16, + "seconds": 90, + "completed": 252, + "cancelled": 134, + "wrong": 0, + "errors": [] + } + ], + "sampled": [ + { + "streams": 1, + "hashes": [ + [ + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c" + ] + ] + }, + { + "streams": 4, + "hashes": [ + [ + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c" + ], + [ + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1" + ], + [ + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d" + ], + [ + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738" + ] + ] + }, + { + "streams": 8, + "hashes": [ + [ + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c" + ], + [ + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1" + ], + [ + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d" + ], + [ + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf", + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf", + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf" + ], + [ + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e", + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e", + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e" + ], + [ + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb", + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb", + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb" + ] + ] + }, + { + "streams": 16, + "hashes": [ + [ + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c" + ], + [ + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1" + ], + [ + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d" + ], + [ + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf", + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf", + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf" + ], + [ + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e", + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e", + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e" + ], + [ + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb", + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb", + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb" + ], + [ + "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c", + "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c", + "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c" + ], + [ + "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b", + "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b", + "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b" + ], + [ + "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2", + "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2", + "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2" + ], + [ + "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3", + "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3", + "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3" + ], + [ + "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de", + "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de", + "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de" + ], + [ + "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74", + "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74", + "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74" + ], + [ + "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8", + "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8", + "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8" + ], + [ + "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8", + "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8", + "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8" + ] + ] + } + ], + "fresh": { + "usage": { + "prompt_tokens": 3605, + "completion_tokens": 64, + "total_tokens": 3669, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 94.76795634919337, + "seconds": 0.664781667, + "prefill_seconds": 4.404813625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 4.405911625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a064a0a2711c", + "token_sha256": "a064a0a2711ce3e5053bd664c3540fc534b5f6044a0aa69b70bce596081d429e", + "min_rows": 2 + }, + "speculative": { + "rounds": 30, + "drafted": 63, + "accepted": 34, + "acceptance_rate": 0.5396825396825397, + "tokens_per_round": 2.1333333333333333 + }, + "wall": 5.071483042091131 + }, + "resume_equal": true + }, + "performance-lfm2": [ + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 134.9239307001102, + "ttft_max": 0.08557750005275011, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 134.91792528327625, + "seconds": 1.890037958, + "prefill_seconds": 0.085051458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.085222875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605043.487071625, + "first": 3605043.572649125, + "last": 3605045.462602958, + "chunks": 205 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 228.88303862287736, + "ttft_max": 0.3615560829639435, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.630612906824005, + "seconds": 4.276327, + "prefill_seconds": 0.091884916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.267615917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605045.4638635, + "first": 3605045.732146916, + "last": 3605050.008385625, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.22032523118682, + "seconds": 4.456458417, + "prefill_seconds": 0.087317125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087474584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605045.463993666, + "first": 3605045.551974833, + "last": 3605050.008344083, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.960532403069806, + "seconds": 4.183034333, + "prefill_seconds": 0.093315625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.360946209, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605045.46392825, + "first": 3605045.825484333, + "last": 3605050.008399416, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.37623247178504, + "seconds": 4.368216125, + "prefill_seconds": 0.088234584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.175705708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605045.463787041, + "first": 3605045.64022925, + "last": 3605050.008372833, + "chunks": 216 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 266.03897119309573, + "ttft_max": 0.7605068329721689, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.09353827701317, + "seconds": 7.479423166, + "prefill_seconds": 0.0945975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2810115, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009457791, + "first": 3605050.291337958, + "last": 3605057.771036541, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.42370539324265, + "seconds": 7.000935167, + "prefill_seconds": 0.096240334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.759415667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009511625, + "first": 3605050.770018458, + "last": 3605057.770923125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.929790863783346, + "seconds": 7.097174625, + "prefill_seconds": 0.096306334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.663202458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009567791, + "first": 3605050.673718083, + "last": 3605057.771226, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.53131103085494, + "seconds": 7.384602333, + "prefill_seconds": 0.094837, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.375764167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009616541, + "first": 3605050.38616375, + "last": 3605057.770763208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.66776444448369, + "seconds": 7.574010458, + "prefill_seconds": 0.093538167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.186413208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.00977625, + "first": 3605050.196752333, + "last": 3605057.770742291, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.98275518635287, + "seconds": 7.289305792, + "prefill_seconds": 0.095362542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.471108958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009734708, + "first": 3605050.481606, + "last": 3605057.771092125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.25701120968319, + "seconds": 7.667556125, + "prefill_seconds": 0.092666459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0928595, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009661, + "first": 3605050.1031765, + "last": 3605057.770667708, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.44872150295576, + "seconds": 7.193489333, + "prefill_seconds": 0.095840875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.566871667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.0093875, + "first": 3605050.577402208, + "last": 3605057.770988958, + "chunks": 229 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 311.1771970060855, + "ttft_max": 1.4885342502966523, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.010677199833463, + "seconds": 12.743196917, + "prefill_seconds": 0.092467916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.458635584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774299166, + "first": 3605058.23469325, + "last": 3605070.977982791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.157114695311563, + "seconds": 12.650620084, + "prefill_seconds": 0.092833334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.551451666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.77475, + "first": 3605058.327514458, + "last": 3605070.978081708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.61350805080629, + "seconds": 12.370529042, + "prefill_seconds": 0.09319175, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.831520625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.7743725, + "first": 3605058.607713041, + "last": 3605070.978289375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.306786367739754, + "seconds": 12.557378375, + "prefill_seconds": 0.093261458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.644682167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.775298666, + "first": 3605058.420757, + "last": 3605070.978255583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.585725861822993, + "seconds": 13.019685959, + "prefill_seconds": 0.091256791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.18242575, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774426458, + "first": 3605057.958124208, + "last": 3605070.978151041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.44945779897582, + "seconds": 13.110905334, + "prefill_seconds": 0.090963583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.091134833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774480916, + "first": 3605057.866821083, + "last": 3605070.977915166, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.77102097803939, + "seconds": 12.276719583, + "prefill_seconds": 0.09383925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.925197417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.775256541, + "first": 3605058.701541375, + "last": 3605070.978314, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.459505653813892, + "seconds": 12.46364425, + "prefill_seconds": 0.093743833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.738352916, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774827583, + "first": 3605058.514517041, + "last": 3605070.978100291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.86659477677835, + "seconds": 12.835616917, + "prefill_seconds": 0.092491125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.366140709, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774540583, + "first": 3605058.1421745, + "last": 3605070.978321666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.724519686382095, + "seconds": 12.928071459, + "prefill_seconds": 0.091558708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.273895042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774587541, + "first": 3605058.049665916, + "last": 3605070.978064916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.424400573077474, + "seconds": 11.902316666, + "prefill_seconds": 0.09373375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.299457542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774637, + "first": 3605059.076021416, + "last": 3605070.978304916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.767101066869646, + "seconds": 11.714927, + "prefill_seconds": 0.0936205, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.486714584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774866916, + "first": 3605059.263401166, + "last": 3605070.978264833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.930350667736132, + "seconds": 12.183264583, + "prefill_seconds": 0.093447667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.01861425, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774904541, + "first": 3605058.795012791, + "last": 3605070.978209041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.092359658728103, + "seconds": 12.08968575, + "prefill_seconds": 0.09357025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.112186208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.77468375, + "first": 3605058.888590125, + "last": 3605070.978239833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.594453198550667, + "seconds": 11.808587958, + "prefill_seconds": 0.093753917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.3930715, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774950875, + "first": 3605059.169768458, + "last": 3605070.978273, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.25698395196336, + "seconds": 11.996057417, + "prefill_seconds": 0.093664167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.205769666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774183791, + "first": 3605058.9823115, + "last": 3605070.978227666, + "chunks": 256 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 157.14330724343966, + "ttft_max": 0.09867354203015566, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.13458006784825, + "seconds": 1.62281275, + "prefill_seconds": 0.098100917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098283125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605070.980492208, + "first": 3605071.07916575, + "last": 3605072.701888375, + "chunks": 145 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 255.09658594530208, + "ttft_max": 0.4042953331954777, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 69.04753394701699, + "seconds": 3.693108, + "prefill_seconds": 0.103401542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.403547042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605072.703041333, + "first": 3605073.107336666, + "last": 3605076.800417541, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.40786452620499, + "seconds": 3.89861375, + "prefill_seconds": 0.09968425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.19807675, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605072.702960125, + "first": 3605072.901815625, + "last": 3605076.800355916, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 67.16709150848577, + "seconds": 3.796502041, + "prefill_seconds": 0.102110291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300229167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605072.703119333, + "first": 3605073.003936041, + "last": 3605076.800587125, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.77710860351041, + "seconds": 3.998299791, + "prefill_seconds": 0.098301959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098602667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605072.702881125, + "first": 3605072.802101625, + "last": 3605076.800324375, + "chunks": 167 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 278.7918956411484, + "ttft_max": 0.8300348329357803, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.85159345841741, + "seconds": 7.316738625, + "prefill_seconds": 0.101190292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101419667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.801830875, + "first": 3605076.904127708, + "last": 3605084.22079525, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.09187846980368, + "seconds": 6.694340375, + "prefill_seconds": 0.104884833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.723694375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.802039125, + "first": 3605077.526568416, + "last": 3605084.221311625, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.70093873857531, + "seconds": 6.588987459, + "prefill_seconds": 0.105350917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.829023, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.8018925, + "first": 3605077.631927333, + "last": 3605084.221138708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.38489569109685, + "seconds": 7.008402667, + "prefill_seconds": 0.10326725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.409696084, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.802100708, + "first": 3605077.212518, + "last": 3605084.220947666, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.85689134832941, + "seconds": 7.111603667, + "prefill_seconds": 0.102814666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.306445875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.802176333, + "first": 3605077.109206125, + "last": 3605084.221413583, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.934623923240295, + "seconds": 6.904090875, + "prefill_seconds": 0.104301042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.513925792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.801991083, + "first": 3605077.316830916, + "last": 3605084.221197833, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.34592633607471, + "seconds": 7.214409875, + "prefill_seconds": 0.102321, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.203729291, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.801947291, + "first": 3605077.006408833, + "last": 3605084.22084075, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.504499161057446, + "seconds": 6.79918425, + "prefill_seconds": 0.10486425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.618898666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.801746666, + "first": 3605077.421678791, + "last": 3605084.220826541, + "chunks": 196 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 312.84454795576346, + "ttft_max": 1.64287120802328, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.680087870793056, + "seconds": 12.330701958, + "prefill_seconds": 0.103426583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.807823167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.223774208, + "first": 3605085.034211333, + "last": 3605097.365090666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.78272141148618, + "seconds": 11.70652625, + "prefill_seconds": 0.104171917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.432023125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.225127125, + "first": 3605085.658580458, + "last": 3605097.365670583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.339825642826977, + "seconds": 12.536980625, + "prefill_seconds": 0.102136875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.6015465, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224564125, + "first": 3605084.827927208, + "last": 3605097.365589625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.17720418879119, + "seconds": 11.498293375, + "prefill_seconds": 0.104055375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.639223542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.22396625, + "first": 3605085.866837458, + "last": 3605097.365761166, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.97827367921699, + "seconds": 11.602367125, + "prefill_seconds": 0.104170167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.5356085, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.223871666, + "first": 3605085.76279375, + "last": 3605097.365196416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.508063343255248, + "seconds": 12.434133625, + "prefill_seconds": 0.102846541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.7043205, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224685291, + "first": 3605084.930803125, + "last": 3605097.365631, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.854728882925233, + "seconds": 12.227442583, + "prefill_seconds": 0.103323125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.911108458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224626083, + "first": 3605085.137543958, + "last": 3605097.365725875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.401224171004174, + "seconds": 11.91520625, + "prefill_seconds": 0.10408875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.223428792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224805583, + "first": 3605085.449904833, + "last": 3605097.365363208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.704330976395816, + "seconds": 12.941317333, + "prefill_seconds": 0.099322166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.197725875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224089458, + "first": 3605084.423510833, + "last": 3605097.36485925, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.85722196718048, + "seconds": 12.841675458, + "prefill_seconds": 0.099694667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.297431583, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224746333, + "first": 3605084.523204583, + "last": 3605097.365503916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.175482988453066, + "seconds": 12.639102625, + "prefill_seconds": 0.10217, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.499395083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.2242065, + "first": 3605084.725776375, + "last": 3605097.365019708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.554265314859858, + "seconds": 13.040633125, + "prefill_seconds": 0.098349833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098607125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224469916, + "first": 3605084.32414025, + "last": 3605097.364759083, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.032647235315533, + "seconds": 12.124008792, + "prefill_seconds": 0.103512084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.014515417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224311875, + "first": 3605085.241059291, + "last": 3605097.365551041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.01370291147283, + "seconds": 12.741270375, + "prefill_seconds": 0.10039225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.397747709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224878875, + "first": 3605084.623611958, + "last": 3605097.365279708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.21588669643829, + "seconds": 12.019294958, + "prefill_seconds": 0.10470325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.119084083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.22439125, + "first": 3605085.345772541, + "last": 3605097.3653205, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.59063720686525, + "seconds": 11.810675042, + "prefill_seconds": 0.104535833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.327713417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.223643833, + "first": 3605085.554434666, + "last": 3605097.365423625, + "chunks": 256 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 135.59348465320016, + "ttft_max": 0.08575020777061582, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 135.5871612465971, + "seconds": 1.880709041, + "prefill_seconds": 0.085088958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.085327083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605102.731765583, + "first": 3605102.817515791, + "last": 3605104.698137125, + "chunks": 205 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 225.62437397965493, + "ttft_max": 0.34385187504813075, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.62565036902639, + "seconds": 4.349631917, + "prefill_seconds": 0.085715541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.256372, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605104.700027083, + "first": 3605104.957033833, + "last": 3605109.306794875, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.49275017434879, + "seconds": 4.435341834, + "prefill_seconds": 0.085297834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.170642833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605104.700094708, + "first": 3605104.871325708, + "last": 3605109.306695375, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.82188035125414, + "seconds": 4.262654375, + "prefill_seconds": 0.086960625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.343317417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605104.700153458, + "first": 3605105.044005333, + "last": 3605109.306778791, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 56.40822558708206, + "seconds": 4.520617292, + "prefill_seconds": 0.085123083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.08533825, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605104.699931625, + "first": 3605104.786006708, + "last": 3605109.306667166, + "chunks": 216 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 239.06832963446098, + "ttft_max": 0.8974757078103721, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 29.885293065711632, + "seconds": 8.532625042, + "prefill_seconds": 0.109820125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.110008416, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308800958, + "first": 3605109.419628375, + "last": 3605117.952543958, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.982160390065538, + "seconds": 7.973194959, + "prefill_seconds": 0.113419125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.66929175, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308735791, + "first": 3605109.979150333, + "last": 3605117.952352875, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 30.682415268592585, + "seconds": 8.310949375, + "prefill_seconds": 0.110996166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.331544792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.309026541, + "first": 3605109.641333083, + "last": 3605117.952246875, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.533558569504233, + "seconds": 8.086623, + "prefill_seconds": 0.112515709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.555912916, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308859125, + "first": 3605109.865747875, + "last": 3605117.952323583, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.100853907471972, + "seconds": 8.199131791, + "prefill_seconds": 0.111869375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.443369, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.30925225, + "first": 3605109.753270708, + "last": 3605117.952603083, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.44146962529053, + "seconds": 7.86030975, + "prefill_seconds": 0.112904125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.782198375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308911541, + "first": 3605110.092090125, + "last": 3605117.952276583, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 30.278069487121737, + "seconds": 8.421937208, + "prefill_seconds": 0.110679667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.220623334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308964416, + "first": 3605109.530324958, + "last": 3605117.952205375, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.9190156055533, + "seconds": 7.746282667, + "prefill_seconds": 0.114026709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.896113292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308652208, + "first": 3605110.206127916, + "last": 3605117.952753625, + "chunks": 222 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 291.00522378471396, + "ttft_max": 1.6490971660241485, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.628776725782256, + "seconds": 12.9911305, + "prefill_seconds": 0.103253458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.129239958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954696041, + "first": 3605119.086046041, + "last": 3605132.077600958, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.876988695385773, + "seconds": 13.5085105, + "prefill_seconds": 0.102846959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.61226975, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955237666, + "first": 3605118.568501666, + "last": 3605132.077354166, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.02239297451555, + "seconds": 13.4052535, + "prefill_seconds": 0.103267709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.715408334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955115041, + "first": 3605118.671778916, + "last": 3605132.077752458, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.473994241940837, + "seconds": 13.094386125, + "prefill_seconds": 0.103817167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.026012584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955405916, + "first": 3605118.982773208, + "last": 3605132.077111625, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.455898082539104, + "seconds": 13.816721292, + "prefill_seconds": 0.10144075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.304203416, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954764791, + "first": 3605118.26023325, + "last": 3605132.076844166, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.32141160864201, + "seconds": 13.918141541, + "prefill_seconds": 0.101377208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.20284425, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954831125, + "first": 3605118.158791583, + "last": 3605132.076891458, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.73436891317599, + "seconds": 13.611347208, + "prefill_seconds": 0.102797667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.50943175, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955307625, + "first": 3605118.465649166, + "last": 3605132.077550958, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.59395354800949, + "seconds": 13.714135584, + "prefill_seconds": 0.102592667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.406669708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955194083, + "first": 3605118.362816333, + "last": 3605132.077146791, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.188910288888422, + "seconds": 14.019531459, + "prefill_seconds": 0.101305167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101564583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954890958, + "first": 3605118.057434458, + "last": 3605132.076825458, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.947034626440626, + "seconds": 12.783855083, + "prefill_seconds": 0.103593084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.336430334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954939458, + "first": 3605119.293258, + "last": 3605132.077039208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.109461251593757, + "seconds": 12.680598292, + "prefill_seconds": 0.103316959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.439643042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955498541, + "first": 3605119.39657675, + "last": 3605132.077682041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.17055943357387, + "seconds": 13.30164625, + "prefill_seconds": 0.103561666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.81862925, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954983666, + "first": 3605118.775327708, + "last": 3605132.077278, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.320957071673337, + "seconds": 13.198103958, + "prefill_seconds": 0.103588, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.922272042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.95503025, + "first": 3605118.8789235, + "last": 3605132.077801333, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.27512247938112, + "seconds": 12.576989375, + "prefill_seconds": 0.103615666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.543083667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.95506975, + "first": 3605119.500232166, + "last": 3605132.077231583, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.78663726380439, + "seconds": 12.887485458, + "prefill_seconds": 0.103617125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.232856375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955829083, + "first": 3605119.189652875, + "last": 3605132.077437666, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.443263874905824, + "seconds": 12.473546375, + "prefill_seconds": 0.103488792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.64631775, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.95461375, + "first": 3605119.603710916, + "last": 3605132.077716833, + "chunks": 229 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 151.98627432059348, + "ttft_max": 0.10828037513419986, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 151.98426635034073, + "seconds": 1.67780525, + "prefill_seconds": 0.10773275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.107901625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605132.079570916, + "first": 3605132.187851291, + "last": 3605133.865634375, + "chunks": 145 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 232.97216394450493, + "ttft_max": 0.4620435410179198, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.24617511786785, + "seconds": 4.377969875, + "prefill_seconds": 0.112383917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.112573583, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605133.868006958, + "first": 3605133.981091541, + "last": 3605138.359214, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.291191598297736, + "seconds": 4.028996667, + "prefill_seconds": 0.118243792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.461524042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605133.868063, + "first": 3605134.330106541, + "last": 3605138.359297, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.809433662600966, + "seconds": 4.263541458, + "prefill_seconds": 0.11442525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.226997416, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605133.868110208, + "first": 3605134.095524666, + "last": 3605138.359019625, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.48683856136229, + "seconds": 4.147229, + "prefill_seconds": 0.11631375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.343276458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605133.867932791, + "first": 3605134.2118565, + "last": 3605138.359159125, + "chunks": 167 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 273.27415594074364, + "ttft_max": 0.904660833068192, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.21827568262825, + "seconds": 7.240558916, + "prefill_seconds": 0.112328125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.335218041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361518583, + "first": 3605138.697389458, + "last": 3605145.938332166, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.7781916277839, + "seconds": 7.12724675, + "prefill_seconds": 0.113313125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.448416083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361304, + "first": 3605138.8107295, + "last": 3605145.93808775, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.35921233541174, + "seconds": 7.01335325, + "prefill_seconds": 0.113931333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.562334125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361365958, + "first": 3605138.924684708, + "last": 3605145.9379155, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.58056725385068, + "seconds": 6.785421792, + "prefill_seconds": 0.11387375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.790247917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361588166, + "first": 3605139.152628875, + "last": 3605145.937980125, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.22085397307286, + "seconds": 6.671750458, + "prefill_seconds": 0.113680292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.903884875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.3616655, + "first": 3605139.266326333, + "last": 3605145.938176333, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.16111921070131, + "seconds": 7.464626625, + "prefill_seconds": 0.110947917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.11120125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361424166, + "first": 3605138.473301333, + "last": 3605145.937860666, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.68028146516437, + "seconds": 7.3528815, + "prefill_seconds": 0.111733208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.222910333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361467666, + "first": 3605138.585063458, + "last": 3605145.938006166, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.960386954623175, + "seconds": 6.899278417, + "prefill_seconds": 0.1140825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.67635775, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361225083, + "first": 3605139.038731333, + "last": 3605145.93793625, + "chunks": 196 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 305.3379633728433, + "ttft_max": 1.633806750178337, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.822927297339806, + "seconds": 12.863892208, + "prefill_seconds": 0.101247709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.59563525, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.940995291, + "first": 3605146.538419875, + "last": 3605159.402413541, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.370879482298406, + "seconds": 11.93212475, + "prefill_seconds": 0.104167625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.526834291, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941094541, + "first": 3605147.470343208, + "last": 3605159.402632, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.142649520347533, + "seconds": 12.659704958, + "prefill_seconds": 0.102535625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.799723584, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941823625, + "first": 3605146.7426835, + "last": 3605159.402381083, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.477246794794308, + "seconds": 12.452845959, + "prefill_seconds": 0.103823583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.006469416, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941530375, + "first": 3605146.949606708, + "last": 3605159.402607833, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.00382553794195, + "seconds": 12.1406455, + "prefill_seconds": 0.104434792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.318557541, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941182916, + "first": 3605147.261812166, + "last": 3605159.402363333, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.668129306462152, + "seconds": 12.965137458, + "prefill_seconds": 0.100130583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.49446125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941742708, + "first": 3605146.437193875, + "last": 3605159.402312875, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.185936302711863, + "seconds": 12.036286542, + "prefill_seconds": 0.104347459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.422695625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.942140416, + "first": 3605147.366185, + "last": 3605159.402436958, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.517372686097072, + "seconds": 13.065283125, + "prefill_seconds": 0.099564083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.394375834, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941583333, + "first": 3605146.337003916, + "last": 3605159.40296525, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.084778292035907, + "seconds": 13.361433709, + "prefill_seconds": 0.09827075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098502, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941264916, + "first": 3605146.040833125, + "last": 3605159.402510166, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.560151994421332, + "seconds": 11.827374875, + "prefill_seconds": 0.104764667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.6315635, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941330583, + "first": 3605147.575137333, + "last": 3605159.402668125, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.649004277684163, + "seconds": 12.34926375, + "prefill_seconds": 0.1036195, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.110109584, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941380125, + "first": 3605147.053187208, + "last": 3605159.403075916, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.82467850167173, + "seconds": 12.245087, + "prefill_seconds": 0.104157708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.21415075, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.94190325, + "first": 3605147.157471791, + "last": 3605159.402822958, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.307930746690957, + "seconds": 12.55667075, + "prefill_seconds": 0.103049709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.902659542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941425291, + "first": 3605146.845705916, + "last": 3605159.402448333, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.225912789847122, + "seconds": 13.263349459, + "prefill_seconds": 0.098078833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196348167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941657666, + "first": 3605146.138916416, + "last": 3605159.402203416, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.980784034931435, + "seconds": 12.762261959, + "prefill_seconds": 0.101674958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.697299833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941471583, + "first": 3605146.640102875, + "last": 3605159.402835125, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.36980007410188, + "seconds": 13.164823541, + "prefill_seconds": 0.0985185, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.294843917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.940892041, + "first": 3605146.237452041, + "last": 3605159.402982875, + "chunks": 203 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 124.69219413734717, + "ttft_max": 0.09309358289465308, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 124.68141557488158, + "seconds": 2.045212583, + "prefill_seconds": 0.092243959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.092521084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 2 + }, + "speculative": { + "rounds": 86, + "drafted": 275, + "accepted": 169, + "acceptance_rate": 0.6145454545454545, + "tokens_per_round": 2.9767441860465116 + }, + "sent": 3605165.7339885, + "first": 3605165.827082083, + "last": 3605167.872117875, + "chunks": 87 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 208.12402438377157, + "ttft_max": 0.4569550408050418, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.27837233945698, + "seconds": 4.78618225, + "prefill_seconds": 0.114795834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.224114042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 46, + "accepted": 37, + "acceptance_rate": 0.8043478260869565, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605167.875097708, + "first": 3605168.099788708, + "last": 3605172.8859015, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 55.99561521394278, + "seconds": 4.553928, + "prefill_seconds": 0.116606292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.456425666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 46, + "accepted": 37, + "acceptance_rate": 0.8043478260869565, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605167.875204875, + "first": 3605168.332159916, + "last": 3605172.885936333, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.03033548440874, + "seconds": 4.900987042, + "prefill_seconds": 0.109144042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.109344458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 46, + "accepted": 37, + "acceptance_rate": 0.8043478260869565, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605167.875152166, + "first": 3605167.985012458, + "last": 3605172.885886541, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.59856064422458, + "seconds": 4.670452792, + "prefill_seconds": 0.115727375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.339796125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 46, + "accepted": 37, + "acceptance_rate": 0.8043478260869565, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605167.875021041, + "first": 3605168.215677, + "last": 3605172.885920875, + "chunks": 219 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 251.6911281708875, + "ttft_max": 0.8647510409355164, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.220280315461096, + "seconds": 7.451721542, + "prefill_seconds": 0.109500625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.75442575, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889201708, + "first": 3605173.644758291, + "last": 3605181.09655925, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.784678873897505, + "seconds": 7.778023417, + "prefill_seconds": 0.107079834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.428348125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.888983666, + "first": 3605173.318376625, + "last": 3605181.096337708, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.24804015841713, + "seconds": 7.669625, + "prefill_seconds": 0.108443959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.536531417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889049125, + "first": 3605173.426806583, + "last": 3605181.096312666, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.729339277725074, + "seconds": 7.342495, + "prefill_seconds": 0.109349667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.863685958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.8893795, + "first": 3605173.754130541, + "last": 3605181.096878333, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.724893118953005, + "seconds": 7.561180375, + "prefill_seconds": 0.1084465, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.644938125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889145291, + "first": 3605173.535275375, + "last": 3605181.09632675, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.33946614750428, + "seconds": 7.885102334, + "prefill_seconds": 0.106294208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.321225417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889298375, + "first": 3605173.211260166, + "last": 3605181.096401833, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.909315309593254, + "seconds": 7.991396792, + "prefill_seconds": 0.11322875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.215124917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889103083, + "first": 3605173.104958333, + "last": 3605181.096496208, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.4635540533525, + "seconds": 8.10461525, + "prefill_seconds": 0.1015975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101866833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.888902375, + "first": 3605172.991705875, + "last": 3605181.096285333, + "chunks": 228 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 313.42934103469173, + "ttft_max": 1.5213572499342263, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.304617493868918, + "seconds": 11.96923625, + "prefill_seconds": 0.094936375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.136720792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.1012335, + "first": 3605182.239094333, + "last": 3605194.208778666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.18264648591888, + "seconds": 12.634616584, + "prefill_seconds": 0.094024083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.471642333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100237125, + "first": 3605181.573688583, + "last": 3605194.208247916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.649907776155853, + "seconds": 11.778341166, + "prefill_seconds": 0.095603625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.327519875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100312666, + "first": 3605182.430033666, + "last": 3605194.208865708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.88634233985453, + "seconds": 12.822870875, + "prefill_seconds": 0.093289083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.283379084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100461416, + "first": 3605181.385283833, + "last": 3605194.208107541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.47560058642282, + "seconds": 11.873940334, + "prefill_seconds": 0.095297208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.231966833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100956208, + "first": 3605182.334405541, + "last": 3605194.208390541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.33497099959012, + "seconds": 12.539973625, + "prefill_seconds": 0.094606625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.566118125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100694333, + "first": 3605181.668279125, + "last": 3605194.208471791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.82723891494671, + "seconds": 11.682650334, + "prefill_seconds": 0.095696292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.423259709, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100372083, + "first": 3605182.525757166, + "last": 3605194.208583583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.742696023980194, + "seconds": 12.916169083, + "prefill_seconds": 0.100347583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.19014025, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100417375, + "first": 3605181.291986041, + "last": 3605194.208053541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.033635098473024, + "seconds": 12.728593625, + "prefill_seconds": 0.094364917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.377678292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.101027916, + "first": 3605181.479648083, + "last": 3605194.208136083, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.971089533527618, + "seconds": 12.159597125, + "prefill_seconds": 0.095635834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.946418208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100824541, + "first": 3605182.048714083, + "last": 3605194.208658458, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.646486217524153, + "seconds": 12.350769875, + "prefill_seconds": 0.0944225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.755228792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100615916, + "first": 3605181.857476375, + "last": 3605194.208165041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.590524741306723, + "seconds": 13.016496667, + "prefill_seconds": 0.089569291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089765958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.10051575, + "first": 3605181.191606166, + "last": 3605194.208012375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.136996552358998, + "seconds": 12.064154875, + "prefill_seconds": 0.095451583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.04185975, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.101147708, + "first": 3605182.144160458, + "last": 3605194.208894166, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.489745150967604, + "seconds": 12.445249959, + "prefill_seconds": 0.094742375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.660845583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.1007485, + "first": 3605181.763080083, + "last": 3605194.20826575, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.80747677102762, + "seconds": 12.255210125, + "prefill_seconds": 0.095575875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.850826625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.10057275, + "first": 3605181.953056416, + "last": 3605194.208488833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.00764844769439, + "seconds": 11.586880834, + "prefill_seconds": 0.095791084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.518658584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100157833, + "first": 3605182.621515083, + "last": 3605194.208810041, + "chunks": 256 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 144.5383970515986, + "ttft_max": 0.09813770884647965, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 144.53195240137714, + "seconds": 1.76431575, + "prefill_seconds": 0.097527583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.097719208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 2 + }, + "speculative": { + "rounds": 78, + "drafted": 285, + "accepted": 177, + "acceptance_rate": 0.6210526315789474, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3605194.211645666, + "first": 3605194.309783375, + "last": 3605196.074020458, + "chunks": 79 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 224.85815545136234, + "ttft_max": 0.46384895825758576, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 56.21441182975977, + "seconds": 4.536203292, + "prefill_seconds": 0.109698167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.109858792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 97, + "accepted": 89, + "acceptance_rate": 0.9175257731958762, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605196.076701458, + "first": 3605196.18695575, + "last": 3605200.72311825, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.964521487528344, + "seconds": 4.182760625, + "prefill_seconds": 0.119314042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.463325, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 97, + "accepted": 89, + "acceptance_rate": 0.9175257731958762, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605196.076561208, + "first": 3605196.540410166, + "last": 3605200.723148791, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.27368123032637, + "seconds": 4.302078, + "prefill_seconds": 0.118036208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.344047292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 97, + "accepted": 89, + "acceptance_rate": 0.9175257731958762, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605196.076630166, + "first": 3605196.421134458, + "last": 3605200.723136666, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.690874674474806, + "seconds": 4.420109791, + "prefill_seconds": 0.116078292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.226017625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 97, + "accepted": 89, + "acceptance_rate": 0.9175257731958762, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605196.076477958, + "first": 3605196.30318375, + "last": 3605200.723097416, + "chunks": 167 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 228.3324076029578, + "ttft_max": 1.0712138330563903, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 29.879040126457436, + "seconds": 8.534410708, + "prefill_seconds": 0.13294175, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.52166925, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726163041, + "first": 3605201.248800291, + "last": 3605209.783159708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 28.54197716851694, + "seconds": 8.934209375, + "prefill_seconds": 0.121683084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.121890709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726410625, + "first": 3605200.848901708, + "last": 3605209.783069583, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 30.860278678874316, + "seconds": 8.263049166, + "prefill_seconds": 0.136530834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.793128791, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.72622, + "first": 3605201.520281958, + "last": 3605209.783206916, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.932770373201418, + "seconds": 7.985527, + "prefill_seconds": 0.1394635, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.070516375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726513375, + "first": 3605201.797727208, + "last": 3605209.7831435, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.384495157857014, + "seconds": 8.125031125, + "prefill_seconds": 0.138017042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.931052709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726452208, + "first": 3605201.658244208, + "last": 3605209.7832445, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 28.98285892631777, + "seconds": 8.798303875, + "prefill_seconds": 0.13589625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.257799375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726282916, + "first": 3605200.984814458, + "last": 3605209.783097333, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 29.42073754897265, + "seconds": 8.667355792, + "prefill_seconds": 0.131006333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.38884425, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726354375, + "first": 3605201.115897041, + "last": 3605209.783179041, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 30.358845572752436, + "seconds": 8.399528875, + "prefill_seconds": 0.134879375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.656546625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.72609275, + "first": 3605201.383693708, + "last": 3605209.783224708, + "chunks": 196 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 282.3386996870639, + "ttft_max": 1.8823581668548286, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.226622070138276, + "seconds": 13.262860167, + "prefill_seconds": 0.119095916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.296116708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787171458, + "first": 3605211.085496583, + "last": 3605224.34824125, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.79779946245813, + "seconds": 14.327613958, + "prefill_seconds": 0.122848958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.231967375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787227958, + "first": 3605210.020624291, + "last": 3605224.34815675, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.055507778747845, + "seconds": 13.38195775, + "prefill_seconds": 0.119004625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.177078458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.788033583, + "first": 3605210.966369083, + "last": 3605224.348362791, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.5742003097195, + "seconds": 13.027352125, + "prefill_seconds": 0.11742125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.531489708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787339583, + "first": 3605211.321101208, + "last": 3605224.348435458, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.24437998196221, + "seconds": 13.976906875, + "prefill_seconds": 0.118000417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.582595542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787291625, + "first": 3605210.371332125, + "last": 3605224.348315875, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.399385883911336, + "seconds": 13.144746, + "prefill_seconds": 0.118168417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.414581041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787738208, + "first": 3605211.203957833, + "last": 3605224.3483335, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.400278522490467, + "seconds": 13.858485875, + "prefill_seconds": 0.118439167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.700956625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787458583, + "first": 3605210.489796333, + "last": 3605224.348448625, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.942747804926768, + "seconds": 14.211870042, + "prefill_seconds": 0.1157755, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.347734791, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787552291, + "first": 3605210.136408458, + "last": 3605224.348429083, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.11232199448928, + "seconds": 12.678794625, + "prefill_seconds": 0.115700042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.880046584, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787384666, + "first": 3605211.669742833, + "last": 3605224.34841975, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.93046255050799, + "seconds": 12.794484792, + "prefill_seconds": 0.116433, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.764330792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787492458, + "first": 3605211.554018, + "last": 3605224.348388208, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.722089689626134, + "seconds": 13.620274458, + "prefill_seconds": 0.119076291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.939076041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787769583, + "first": 3605210.728119333, + "last": 3605224.348404166, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.750736349295554, + "seconds": 12.910911041, + "prefill_seconds": 0.116433375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.647919833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787423583, + "first": 3605211.437522208, + "last": 3605224.3483775, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.091644716210116, + "seconds": 14.09490425, + "prefill_seconds": 0.116940458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.464571833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787609208, + "first": 3605210.253391583, + "last": 3605224.348343875, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.559834143429526, + "seconds": 13.739346916, + "prefill_seconds": 0.119137375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.819974375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787643166, + "first": 3605210.608930375, + "last": 3605224.34844175, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.646478088315806, + "seconds": 14.450475541, + "prefill_seconds": 0.109028625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.109224, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787675125, + "first": 3605209.89771975, + "last": 3605224.348114458, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.88748538643221, + "seconds": 13.501003166, + "prefill_seconds": 0.119320625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.058378042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787071333, + "first": 3605210.847375166, + "last": 3605224.348277583, + "chunks": 254 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 135.3705299959295, + "ttft_max": 0.08670658292248845, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 135.36170933161313, + "seconds": 1.883841459, + "prefill_seconds": 0.085781333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.086096792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605230.084579083, + "first": 3605230.171285666, + "last": 3605232.055004375, + "chunks": 205 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 247.18854085067397, + "ttft_max": 0.3422554158605635, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 64.4663127278345, + "seconds": 3.955554292, + "prefill_seconds": 0.085400125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.255194791, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605232.05815025, + "first": 3605232.314051291, + "last": 3605236.269604041, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.10377137843549, + "seconds": 4.040962916, + "prefill_seconds": 0.085076667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169815167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605232.0582255, + "first": 3605232.228662958, + "last": 3605236.269935791, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.80205211183789, + "seconds": 4.126076583, + "prefill_seconds": 0.084549458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.084750125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605232.058285291, + "first": 3605232.143530916, + "last": 3605236.269508, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.90286679271894, + "seconds": 3.869330917, + "prefill_seconds": 0.086262041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.341408667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605232.058066125, + "first": 3605232.400321541, + "last": 3605236.269820875, + "chunks": 216 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 268.44417944890495, + "ttft_max": 0.7584767080843449, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.30311403855571, + "seconds": 7.223158833, + "prefill_seconds": 0.095596542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.468107459, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273302125, + "first": 3605236.74224275, + "last": 3605243.965309791, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.268515877396204, + "seconds": 7.0308915, + "prefill_seconds": 0.09632875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.66032425, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273358083, + "first": 3605236.9345115, + "last": 3605243.965373625, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.97195388726913, + "seconds": 7.50619175, + "prefill_seconds": 0.093056167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.185104417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.27363875, + "first": 3605236.459160666, + "last": 3605243.965230541, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.39914261753111, + "seconds": 7.412975458, + "prefill_seconds": 0.093215666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.278315125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.27341825, + "first": 3605236.552370416, + "last": 3605243.965326041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.842132802854294, + "seconds": 7.318725333, + "prefill_seconds": 0.09426725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.372486833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.27355425, + "first": 3605236.646647666, + "last": 3605243.965269583, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.55590475435463, + "seconds": 7.599258666, + "prefill_seconds": 0.091830958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.092044417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273475458, + "first": 3605236.366028375, + "last": 3605243.965252666, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.77833657849277, + "seconds": 7.127217875, + "prefill_seconds": 0.095941375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.563977417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273594833, + "first": 3605236.83817375, + "last": 3605243.965293083, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.776712264682, + "seconds": 6.933735625, + "prefill_seconds": 0.097161791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.757486292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273225458, + "first": 3605237.031702166, + "last": 3605243.965334, + "chunks": 229 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 319.2847676851127, + "ttft_max": 1.4483123333193362, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.80600240787437, + "seconds": 11.694027875, + "prefill_seconds": 0.091334042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.173616291, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969443166, + "first": 3605245.145292416, + "last": 3605256.839270791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.150983325961008, + "seconds": 11.511904291, + "prefill_seconds": 0.091012292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.355748375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.96950325, + "first": 3605245.327496291, + "last": 3605256.839313541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.145950469632904, + "seconds": 12.0590465, + "prefill_seconds": 0.090774708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.808576667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.970349458, + "first": 3605244.780217958, + "last": 3605256.839328166, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.2376016783258, + "seconds": 12.600307292, + "prefill_seconds": 0.089279583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.267583958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969966791, + "first": 3605244.238831666, + "last": 3605256.839117791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.833228656571702, + "seconds": 12.2400615, + "prefill_seconds": 0.090448792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.627604958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.970547625, + "first": 3605244.599190125, + "last": 3605256.839249541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.98798123254718, + "seconds": 12.14981075, + "prefill_seconds": 0.090244583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.717820292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.970049083, + "first": 3605244.689422625, + "last": 3605256.83936075, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.68041589249006, + "seconds": 12.330506375, + "prefill_seconds": 0.09017325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.537172167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969576333, + "first": 3605244.508701625, + "last": 3605256.839198958, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.955637312919926, + "seconds": 12.778344084, + "prefill_seconds": 0.089494417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089734333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969684041, + "first": 3605244.060805875, + "last": 3605256.839061666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.095216256935384, + "seconds": 12.68958725, + "prefill_seconds": 0.088746, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.178434625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.96963825, + "first": 3605244.149560791, + "last": 3605256.8390755, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.637063857646872, + "seconds": 11.785332875, + "prefill_seconds": 0.091055709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.08228675, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969914375, + "first": 3605245.053956541, + "last": 3605256.839345041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.32804931103493, + "seconds": 11.420612542, + "prefill_seconds": 0.091313042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.446868375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.97046325, + "first": 3605245.418775583, + "last": 3605256.83936725, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.383060046456592, + "seconds": 12.5103885, + "prefill_seconds": 0.089931041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.357453166, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969826875, + "first": 3605244.32876, + "last": 3605256.839146, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.9772229504082, + "seconds": 11.602921833, + "prefill_seconds": 0.09111725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.264695667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.97027275, + "first": 3605245.236458291, + "last": 3605256.839352625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.530285809656654, + "seconds": 12.420674625, + "prefill_seconds": 0.089750042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.447048958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.970631333, + "first": 3605244.41854875, + "last": 3605256.83918125, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.471178787040312, + "seconds": 11.876385667, + "prefill_seconds": 0.091522541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.99120025, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969740833, + "first": 3605244.962909166, + "last": 3605256.8392915, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.307010309878976, + "seconds": 11.967892083, + "prefill_seconds": 0.091147875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.899696625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969366083, + "first": 3605244.871349166, + "last": 3605256.839337791, + "chunks": 256 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 157.47416781759355, + "ttft_max": 0.0967360413633287, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.4648286095686, + "seconds": 1.61940925, + "prefill_seconds": 0.096069542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.096265166, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605256.844516, + "first": 3605256.941252041, + "last": 3605258.56056525, + "chunks": 145 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 254.6708458908824, + "ttft_max": 0.3996269996277988, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.28964545138331, + "seconds": 3.905672917, + "prefill_seconds": 0.099539084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.19903375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605258.564169375, + "first": 3605258.763831125, + "last": 3605262.669454333, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 66.9946382614655, + "seconds": 3.80627475, + "prefill_seconds": 0.099390625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.298365042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605258.564251458, + "first": 3605258.863197083, + "last": 3605262.669443208, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 68.81561334793373, + "seconds": 3.705554417, + "prefill_seconds": 0.100711833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.399111875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605258.564329916, + "first": 3605258.963956916, + "last": 3605262.669422791, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.66673319241069, + "seconds": 4.005231417, + "prefill_seconds": 0.099261291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.09953825, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605258.564030833, + "first": 3605258.664284458, + "last": 3605262.669395041, + "chunks": 167 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 285.70738648598456, + "ttft_max": 0.8014092496596277, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.716172818794945, + "seconds": 7.13962275, + "prefill_seconds": 0.098861958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0990905, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.67330575, + "first": 3605262.773175375, + "last": 3605269.912740333, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.99465037162661, + "seconds": 6.539358542, + "prefill_seconds": 0.10073975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.699211458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673546166, + "first": 3605263.37350375, + "last": 3605269.912882291, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.82942755620118, + "seconds": 6.740784, + "prefill_seconds": 0.100308, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.497804958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673493833, + "first": 3605263.172063958, + "last": 3605269.91280275, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.274715233487235, + "seconds": 6.841098541, + "prefill_seconds": 0.099807333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.397588125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673645791, + "first": 3605263.07171825, + "last": 3605269.9129505, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.73872080653342, + "seconds": 6.940905791, + "prefill_seconds": 0.099730791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.297757625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673439583, + "first": 3605262.971927291, + "last": 3605269.912967, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.21838064353387, + "seconds": 7.040624, + "prefill_seconds": 0.098990417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198081833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673371833, + "first": 3605262.872175791, + "last": 3605269.913055333, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 39.61007249398499, + "seconds": 6.437756458, + "prefill_seconds": 0.101606083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.800571709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673717208, + "first": 3605263.475126458, + "last": 3605269.913224208, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.403006695548854, + "seconds": 6.640105084, + "prefill_seconds": 0.100696167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.598439417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673213166, + "first": 3605263.272753958, + "last": 3605269.913347791, + "chunks": 196 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 303.7043595048409, + "ttft_max": 1.6232123747467995, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.707553939051333, + "seconds": 12.939200917, + "prefill_seconds": 0.100283042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.591923375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917221625, + "first": 3605270.510649541, + "last": 3605283.449745416, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.1210335801187, + "seconds": 13.336099167, + "prefill_seconds": 0.097765375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.195100708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917575666, + "first": 3605270.113639666, + "last": 3605283.44979175, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.01970233116774, + "seconds": 12.737452125, + "prefill_seconds": 0.101138208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.793557834, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917355333, + "first": 3605270.712412625, + "last": 3605283.449932916, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.346740438964275, + "seconds": 12.532719959, + "prefill_seconds": 0.103530875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.998317875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917632041, + "first": 3605270.917182083, + "last": 3605283.449850291, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.693519640614777, + "seconds": 12.322698334, + "prefill_seconds": 0.10384725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.208413917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.918093333, + "first": 3605271.127308625, + "last": 3605283.449959291, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.86200798449543, + "seconds": 12.838581084, + "prefill_seconds": 0.100614541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.692508416, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917294666, + "first": 3605270.611324958, + "last": 3605283.449966333, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.26308732203688, + "seconds": 13.237753416, + "prefill_seconds": 0.09834925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.293452084, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917457625, + "first": 3605270.211989041, + "last": 3605283.449669833, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.180066868031723, + "seconds": 12.636231667, + "prefill_seconds": 0.101215833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.894750166, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917404291, + "first": 3605270.813657875, + "last": 3605283.449834666, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.228878666832493, + "seconds": 12.011939208, + "prefill_seconds": 0.102896834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.518830125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917793791, + "first": 3605271.437999583, + "last": 3605283.44995075, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.412562914304186, + "seconds": 11.908896708, + "prefill_seconds": 0.103050708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.621631667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917860541, + "first": 3605271.541072916, + "last": 3605283.449972708, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.55593888604215, + "seconds": 13.039517125, + "prefill_seconds": 0.099267583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.491657291, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917944833, + "first": 3605270.410338416, + "last": 3605283.449818, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.981866052192267, + "seconds": 13.433874167, + "prefill_seconds": 0.097258833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.097454292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917525875, + "first": 3605270.015855375, + "last": 3605283.449766875, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.87036352214025, + "seconds": 12.218282625, + "prefill_seconds": 0.104461041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.312728458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917714166, + "first": 3605271.231622583, + "last": 3605283.449863208, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.520543693562686, + "seconds": 12.426571333, + "prefill_seconds": 0.106122416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.104378167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917754708, + "first": 3605271.023299541, + "last": 3605283.449915125, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.048586514158238, + "seconds": 12.11482775, + "prefill_seconds": 0.103449667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.416263459, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917914916, + "first": 3605271.335092041, + "last": 3605283.449897208, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.408236402670443, + "seconds": 13.13875175, + "prefill_seconds": 0.099055875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.392426458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917140958, + "first": 3605270.311076083, + "last": 3605283.449698625, + "chunks": 220 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 126.78640117388437, + "ttft_max": 0.09193537477403879, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 126.76524222550795, + "seconds": 2.011592417, + "prefill_seconds": 0.090170916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.090489583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 2 + }, + "speculative": { + "rounds": 85, + "drafted": 283, + "accepted": 170, + "acceptance_rate": 0.6007067137809188, + "tokens_per_round": 3.011764705882353 + }, + "sent": 3605290.036222666, + "first": 3605290.128158041, + "last": 3605292.13941475, + "chunks": 86 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 200.43096587895258, + "ttft_max": 0.4380846251733601, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.650528654504505, + "seconds": 5.034498292, + "prefill_seconds": 0.104293291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.104680292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 197, + "drafted": 73, + "accepted": 58, + "acceptance_rate": 0.7945205479452054, + "tokens_per_round": 1.299492385786802 + }, + "sent": 3605292.143276458, + "first": 3605292.248608291, + "last": 3605297.2830055, + "chunks": 198 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.474920914161814, + "seconds": 4.859464208, + "prefill_seconds": 0.111324375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.325527875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 200, + "drafted": 74, + "accepted": 55, + "acceptance_rate": 0.7432432432432432, + "tokens_per_round": 1.28 + }, + "sent": 3605292.143338916, + "first": 3605292.469470583, + "last": 3605297.328782541, + "chunks": 201 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.61383015029026, + "seconds": 4.756235458, + "prefill_seconds": 0.112080791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.43766275, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 201, + "drafted": 69, + "accepted": 54, + "acceptance_rate": 0.782608695652174, + "tokens_per_round": 1.2736318407960199 + }, + "sent": 3605292.143407208, + "first": 3605292.581491833, + "last": 3605297.337642291, + "chunks": 202 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.42472493865987, + "seconds": 4.958704209, + "prefill_seconds": 0.109472208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.214242583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 199, + "drafted": 71, + "accepted": 56, + "acceptance_rate": 0.7887323943661971, + "tokens_per_round": 1.2864321608040201 + }, + "sent": 3605292.143192375, + "first": 3605292.358079333, + "last": 3605297.316705916, + "chunks": 200 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 255.46984362436277, + "ttft_max": 0.9172884998843074, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.96757134858678, + "seconds": 7.292471, + "prefill_seconds": 0.115811167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.800858792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341384333, + "first": 3605298.143214708, + "last": 3605305.435554958, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.53055484152029, + "seconds": 7.176921417, + "prefill_seconds": 0.115590667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.91638425, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341507458, + "first": 3605298.258795958, + "last": 3605305.435580625, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.38492431371953, + "seconds": 7.638178167, + "prefill_seconds": 0.114445042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.455078458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341628541, + "first": 3605297.797422375, + "last": 3605305.435521375, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.934024719695735, + "seconds": 7.985213334, + "prefill_seconds": 0.107942584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.108172083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341680875, + "first": 3605297.450301958, + "last": 3605305.435538041, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.42087377624303, + "seconds": 7.408295375, + "prefill_seconds": 0.11484875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.685039542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341740041, + "first": 3605298.027373458, + "last": 3605305.43556775, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.895674014438306, + "seconds": 7.523083916, + "prefill_seconds": 0.115094834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.570190875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341450916, + "first": 3605297.912514875, + "last": 3605305.435588875, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.892222827396125, + "seconds": 7.7525925, + "prefill_seconds": 0.113420959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.340682666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341554, + "first": 3605297.682979916, + "last": 3605305.435493916, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.41796563335079, + "seconds": 7.866008709, + "prefill_seconds": 0.119201834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.227344958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.34129825, + "first": 3605297.569545375, + "last": 3605305.435476666, + "chunks": 228 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 313.8774581034392, + "ttft_max": 1.5110153751447797, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.060272004503304, + "seconds": 12.711692042, + "prefill_seconds": 0.093187083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.374716458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.44135625, + "first": 3605305.817483791, + "last": 3605318.530290208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.67025183008689, + "seconds": 12.336569583, + "prefill_seconds": 0.094140792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.749637917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441405125, + "first": 3605306.192615291, + "last": 3605318.529260125, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.513701085871652, + "seconds": 12.430716375, + "prefill_seconds": 0.094110208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.655543625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.442013791, + "first": 3605306.098466166, + "last": 3605318.529441541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.829214784946384, + "seconds": 12.242420208, + "prefill_seconds": 0.094183209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.843790334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441461916, + "first": 3605306.2868475, + "last": 3605318.530216875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.359571456579687, + "seconds": 12.524821583, + "prefill_seconds": 0.093567333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.561439209, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441520416, + "first": 3605306.004342125, + "last": 3605318.530478416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.495463716658133, + "seconds": 11.862968083, + "prefill_seconds": 0.095485417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.223262791, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441578041, + "first": 3605306.666461125, + "last": 3605318.529598458, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.02659359839882, + "seconds": 11.576914917, + "prefill_seconds": 0.095079667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.509074917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441627291, + "first": 3605306.952642666, + "last": 3605318.529668291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.156408017755954, + "seconds": 12.05308575, + "prefill_seconds": 0.094809959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.033162541, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441862375, + "first": 3605306.476218875, + "last": 3605318.52974675, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.99136372992379, + "seconds": 12.147852959, + "prefill_seconds": 0.094577583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.938381959, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.44167525, + "first": 3605306.381410791, + "last": 3605318.529809208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.669564145303198, + "seconds": 11.767657083, + "prefill_seconds": 0.095331208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.318576959, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441896666, + "first": 3605306.76181175, + "last": 3605318.529422416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.619602263921294, + "seconds": 12.997205375, + "prefill_seconds": 0.088972792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089189584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.44171625, + "first": 3605305.531774958, + "last": 3605318.528930666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.847292069824867, + "seconds": 11.671927083, + "prefill_seconds": 0.095725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.414164333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441935625, + "first": 3605306.857545041, + "last": 3605318.529885, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.32386163636074, + "seconds": 11.958434375, + "prefill_seconds": 0.094735083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.127798667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441755666, + "first": 3605306.57101725, + "last": 3605318.530401416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.914469759301284, + "seconds": 12.804759709, + "prefill_seconds": 0.092802833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281539584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441971875, + "first": 3605305.724298875, + "last": 3605318.529047875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.208637605857678, + "seconds": 12.618366709, + "prefill_seconds": 0.093288458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.467903583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441798791, + "first": 3605305.910813125, + "last": 3605318.529313708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.77116191646302, + "seconds": 12.897572792, + "prefill_seconds": 0.099672334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188832667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441279833, + "first": 3605305.63146675, + "last": 3605318.528985041, + "chunks": 256 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 147.83629271145213, + "ttft_max": 0.096926209051162, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 147.82846866597168, + "seconds": 1.724972208, + "prefill_seconds": 0.096228583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.096440459, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 2 + }, + "speculative": { + "rounds": 78, + "drafted": 284, + "accepted": 177, + "acceptance_rate": 0.6232394366197183, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3605318.535977041, + "first": 3605318.63290325, + "last": 3605320.357784166, + "chunks": 79 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 220.9503576822706, + "ttft_max": 0.4242655001580715, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.40146559139531, + "seconds": 4.292823375, + "prefill_seconds": 0.109193292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.423730833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 3605320.361777708, + "first": 3605320.786043208, + "last": 3605325.078767416, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 55.23800969974208, + "seconds": 4.616386459, + "prefill_seconds": 0.099986875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.100170916, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 3605320.361634458, + "first": 3605320.462448, + "last": 3605325.078736916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.92797484039848, + "seconds": 4.402018208, + "prefill_seconds": 0.108133708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.314518667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 3605320.361707958, + "first": 3605320.67684525, + "last": 3605325.078869583, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 56.53916666194564, + "seconds": 4.510147833, + "prefill_seconds": 0.106229667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206412666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 3605320.361551875, + "first": 3605320.568685541, + "last": 3605325.0787525, + "chunks": 253 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 257.0465470267213, + "ttft_max": 0.8670476251281798, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.559741330359046, + "seconds": 7.171030791, + "prefill_seconds": 0.110246708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.865938416, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.082999916, + "first": 3605325.950047541, + "last": 3605333.120935583, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.0487811674307, + "seconds": 7.715867, + "prefill_seconds": 0.107012416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.321258417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.08327425, + "first": 3605325.405153541, + "last": 3605333.1209695, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.0007043812591, + "seconds": 7.499844625, + "prefill_seconds": 0.108330958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.537242959, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083354583, + "first": 3605325.621150041, + "last": 3605333.120979291, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.59696730918858, + "seconds": 7.822813625, + "prefill_seconds": 0.113354958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.214276167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083070083, + "first": 3605325.298106916, + "last": 3605333.120946875, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.02128968234008, + "seconds": 7.28128525, + "prefill_seconds": 0.109204875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.755724292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083523375, + "first": 3605325.83974725, + "last": 3605333.121043833, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.51667722496765, + "seconds": 7.60815275, + "prefill_seconds": 0.107665042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.428864333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083146, + "first": 3605325.512829125, + "last": 3605333.120899958, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.50375520787656, + "seconds": 7.390499917, + "prefill_seconds": 0.109379, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.6465175, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083463166, + "first": 3605325.730562708, + "last": 3605333.121033916, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.13133897047426, + "seconds": 7.936177208, + "prefill_seconds": 0.100699916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.100950125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.082914666, + "first": 3605325.184738041, + "last": 3605333.121023791, + "chunks": 217 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 263.9872023279576, + "ttft_max": 2.16144795762375, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 16.500083985330427, + "seconds": 15.454466791, + "prefill_seconds": 0.122913333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1231215, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126206875, + "first": 3605333.25041925, + "last": 3605348.704820291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.817667675727197, + "seconds": 13.551094875, + "prefill_seconds": 0.134893458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.025968417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126846, + "first": 3605335.154130541, + "last": 3605348.705543875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 16.9412854102989, + "seconds": 15.051986542, + "prefill_seconds": 0.133306958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.525254, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126274166, + "first": 3605333.6529565, + "last": 3605348.704856916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.577823638472832, + "seconds": 14.506915375, + "prefill_seconds": 0.136994667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.070291541, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126759583, + "first": 3605334.198122, + "last": 3605348.704980125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.00578384558076, + "seconds": 13.416968333, + "prefill_seconds": 0.13424025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.160254292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126926333, + "first": 3605335.288374291, + "last": 3605348.705384041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.747479878980165, + "seconds": 14.368237166, + "prefill_seconds": 0.138685917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.208875292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126330583, + "first": 3605334.33682525, + "last": 3605348.705066833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.09438622618824, + "seconds": 14.917177875, + "prefill_seconds": 0.134858, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.660051667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126624666, + "first": 3605333.787864, + "last": 3605348.704940416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.252081909394978, + "seconds": 14.780824792, + "prefill_seconds": 0.136416833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.796385917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126385708, + "first": 3605333.924240208, + "last": 3605348.7050225, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.41324414474544, + "seconds": 14.644026, + "prefill_seconds": 0.136864667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.933301083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126715541, + "first": 3605334.061117541, + "last": 3605348.705056041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.092909645749767, + "seconds": 14.093918833, + "prefill_seconds": 0.137012834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.483268125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126442833, + "first": 3605334.611230625, + "last": 3605348.705081875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 16.792604524165665, + "seconds": 15.185256083, + "prefill_seconds": 0.131768125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.392004542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126482833, + "first": 3605333.519721708, + "last": 3605348.705042458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.449357599261333, + "seconds": 13.821619459, + "prefill_seconds": 0.135470416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.755477959, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126891791, + "first": 3605334.883549125, + "last": 3605348.705171125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.270250536285104, + "seconds": 13.957115667, + "prefill_seconds": 0.13683125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.620021208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126673666, + "first": 3605334.748088833, + "last": 3605348.705274708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 16.648131928168212, + "seconds": 15.317033833, + "prefill_seconds": 0.137435125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.260257041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126563791, + "first": 3605333.387854375, + "last": 3605348.704874083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.91874193182425, + "seconds": 14.230909791, + "prefill_seconds": 0.137384084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.346198542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126526375, + "first": 3605334.474237, + "last": 3605348.705713916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.632244499725477, + "seconds": 13.685951792, + "prefill_seconds": 0.135675834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.891081125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126121875, + "first": 3605335.019221583, + "last": 3605348.705492625, + "chunks": 252 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 135.21683007811936, + "ttft_max": 0.0871472917497158, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 135.2024288303004, + "seconds": 1.886060792, + "prefill_seconds": 0.086245667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.086543083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605354.644582916, + "first": 3605354.731730208, + "last": 3605356.617590125, + "chunks": 205 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 241.76743130633574, + "ttft_max": 0.35046541690826416, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.718282989438244, + "seconds": 4.131676833, + "prefill_seconds": 0.086965875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.17304075, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605356.622275875, + "first": 3605356.795996958, + "last": 3605360.927636875, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.05896439746511, + "seconds": 4.04383425, + "prefill_seconds": 0.087845292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.26081375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605356.622413541, + "first": 3605356.883842, + "last": 3605360.927687291, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 64.47625109491958, + "seconds": 3.954944583, + "prefill_seconds": 0.08896475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.349774417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605356.622349708, + "first": 3605356.972815125, + "last": 3605360.927937083, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.44555424989418, + "seconds": 4.218672542, + "prefill_seconds": 0.085891709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.086074041, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605356.622183791, + "first": 3605356.709006541, + "last": 3605360.927656833, + "chunks": 216 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 270.7456803820994, + "ttft_max": 0.7304916670545936, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.10999689684354, + "seconds": 7.262888708, + "prefill_seconds": 0.091176541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.361503833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933122416, + "first": 3605361.295646916, + "last": 3605368.558554916, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.25366513350512, + "seconds": 7.444458834, + "prefill_seconds": 0.09023775, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.179989584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933404833, + "first": 3605361.114011958, + "last": 3605368.558453083, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.55645373853483, + "seconds": 7.171693833, + "prefill_seconds": 0.091205792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.452702834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933335958, + "first": 3605361.386909875, + "last": 3605368.558520791, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.843515292583646, + "seconds": 7.534678292, + "prefill_seconds": 0.089659083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089863125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933188916, + "first": 3605361.023834791, + "last": 3605368.558379916, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.49754156803731, + "seconds": 6.986771959, + "prefill_seconds": 0.092288, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.637563583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933269291, + "first": 3605361.571819208, + "last": 3605368.558564291, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.9856298318258, + "seconds": 6.894569625, + "prefill_seconds": 0.09220275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.729732584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933553041, + "first": 3605361.664044708, + "last": 3605368.558581041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.67458892637427, + "seconds": 7.354088625, + "prefill_seconds": 0.090460375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.270407417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933460708, + "first": 3605361.204480708, + "last": 3605368.558538208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.021696924128804, + "seconds": 7.079066834, + "prefill_seconds": 0.092644084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.545317208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.93303275, + "first": 3605361.479543083, + "last": 3605368.558492, + "chunks": 229 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 317.14545460069326, + "ttft_max": 1.5035453750751913, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.264583340348953, + "seconds": 12.583530375, + "prefill_seconds": 0.094196833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.373004334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564750375, + "first": 3605368.939365208, + "last": 3605381.523284958, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.114033885301385, + "seconds": 12.677715542, + "prefill_seconds": 0.093041542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.278851875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565608166, + "first": 3605368.845154833, + "last": 3605381.523831416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.08067244265804, + "seconds": 11.548561334, + "prefill_seconds": 0.093428917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.407884875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565175666, + "first": 3605369.975220458, + "last": 3605381.523954666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.581051025094403, + "seconds": 12.390037792, + "prefill_seconds": 0.100268333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.566598, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564819791, + "first": 3605369.133161083, + "last": 3605381.523207875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.221400404997944, + "seconds": 12.016172125, + "prefill_seconds": 0.093754625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.940406041, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.56523525, + "first": 3605369.507174916, + "last": 3605381.523527625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.26102090220382, + "seconds": 11.45500025, + "prefill_seconds": 0.093674042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.501510833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565353291, + "first": 3605370.068898666, + "last": 3605381.523995208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.823640472034473, + "seconds": 12.863429417, + "prefill_seconds": 0.092897375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.093108458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564889291, + "first": 3605368.659426416, + "last": 3605381.523334791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.05599916202474, + "seconds": 12.110562792, + "prefill_seconds": 0.093667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.84692175, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565460083, + "first": 3605369.413415208, + "last": 3605381.524185916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.967498850440553, + "seconds": 12.770753208, + "prefill_seconds": 0.092665459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.185828791, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564947375, + "first": 3605368.752121291, + "last": 3605381.522796333, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.387795624969122, + "seconds": 11.922687334, + "prefill_seconds": 0.093571708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.033904875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565401291, + "first": 3605369.6007535, + "last": 3605381.524110041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.736696626974393, + "seconds": 12.297040584, + "prefill_seconds": 0.093499584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.660084125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565037166, + "first": 3605369.226532875, + "last": 3605381.523677875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.894685794925884, + "seconds": 12.204060042, + "prefill_seconds": 0.093236083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.753312333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564991208, + "first": 3605369.319736166, + "last": 3605381.523785666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.415943287029275, + "seconds": 12.490238458, + "prefill_seconds": 0.093332875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.466354083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565313833, + "first": 3605369.032799916, + "last": 3605381.523712625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.556892300832946, + "seconds": 11.829163334, + "prefill_seconds": 0.093681209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.127521958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565076666, + "first": 3605369.694434791, + "last": 3605381.524069916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.729100870895973, + "seconds": 11.73541425, + "prefill_seconds": 0.093859833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.221267625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565120791, + "first": 3605369.78833975, + "last": 3605381.524150666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.903448234267138, + "seconds": 11.642002541, + "prefill_seconds": 0.09342325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.314510833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564657416, + "first": 3605369.881750375, + "last": 3605381.523902958, + "chunks": 256 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 155.2756224781027, + "ttft_max": 0.09816666599363089, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.26933117741075, + "seconds": 1.642307583, + "prefill_seconds": 0.097623791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.097776958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605381.528660625, + "first": 3605381.626827291, + "last": 3605383.269068333, + "chunks": 145 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 261.3078294249977, + "ttft_max": 0.3997202501632273, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 67.02797961517537, + "seconds": 3.804381416, + "prefill_seconds": 0.099036, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1972735, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605383.273448833, + "first": 3605383.471414916, + "last": 3605387.275758833, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 68.83537927939683, + "seconds": 3.704490375, + "prefill_seconds": 0.09988475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.297147083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605383.273581875, + "first": 3605383.571316625, + "last": 3605387.275774583, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 70.7808129870301, + "seconds": 3.60267125, + "prefill_seconds": 0.101852208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.398936292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605383.273515041, + "first": 3605383.673235291, + "last": 3605387.275792125, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.32720632146649, + "seconds": 3.903427291, + "prefill_seconds": 0.097979833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098211333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605383.273371041, + "first": 3605383.372349958, + "last": 3605387.275720166, + "chunks": 167 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 280.2334342413751, + "ttft_max": 0.8117147092707455, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.084801340102665, + "seconds": 6.876132291, + "prefill_seconds": 0.102005167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.501668542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280753, + "first": 3605387.783466291, + "last": 3605394.659875291, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.228965469386054, + "seconds": 6.670334833, + "prefill_seconds": 0.102883166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.70730875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.281004958, + "first": 3605387.989273833, + "last": 3605394.659580416, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.54287143821144, + "seconds": 6.978105167, + "prefill_seconds": 0.101316292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.39971725, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280939791, + "first": 3605387.681480041, + "last": 3605394.659770625, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.64820566621007, + "seconds": 6.773231167, + "prefill_seconds": 0.102894541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.604616459, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280877833, + "first": 3605387.886363625, + "last": 3605394.659825291, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.83195706222774, + "seconds": 6.566756334, + "prefill_seconds": 0.103611958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.81094475, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.281200166, + "first": 3605388.092914875, + "last": 3605394.659544541, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.516121262368586, + "seconds": 7.179838083, + "prefill_seconds": 0.09941125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198144292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280812791, + "first": 3605387.479661916, + "last": 3605394.659483416, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.01983336947377, + "seconds": 7.079433083, + "prefill_seconds": 0.100446917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.29840925, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.281063625, + "first": 3605387.580137125, + "last": 3605394.6596125, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.03117754607157, + "seconds": 7.279230042, + "prefill_seconds": 0.098551667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098811084, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280672125, + "first": 3605387.38023, + "last": 3605394.659435708, + "chunks": 196 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 278.70302133474246, + "ttft_max": 1.760789040941745, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.50306570402351, + "seconds": 13.7815, + "prefill_seconds": 0.110498333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.959904333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665737375, + "first": 3605395.626808416, + "last": 3605409.408287583, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.808313204366144, + "seconds": 13.557834625, + "prefill_seconds": 0.112213583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.183590458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665193875, + "first": 3605395.850536708, + "last": 3605409.40838575, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.96723081167307, + "seconds": 13.444239833, + "prefill_seconds": 0.113519625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.297130625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665262833, + "first": 3605395.964034166, + "last": 3605409.408241791, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.129585362089532, + "seconds": 13.330137333, + "prefill_seconds": 0.114137583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.411010958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.666038291, + "first": 3605396.078187125, + "last": 3605409.408339708, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.542866690280768, + "seconds": 14.535822708, + "prefill_seconds": 0.103120084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.205699625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665567166, + "first": 3605394.872240125, + "last": 3605409.408193541, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.46713693925592, + "seconds": 13.098998625, + "prefill_seconds": 0.11613475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.642015666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.666172666, + "first": 3605396.309409916, + "last": 3605409.408378, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.801062083252255, + "seconds": 14.324987959, + "prefill_seconds": 0.105807542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.416487666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665355333, + "first": 3605395.083124083, + "last": 3605409.408067333, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.654002241477198, + "seconds": 13.669988708, + "prefill_seconds": 0.111501334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.071457708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.66611875, + "first": 3605395.738288291, + "last": 3605409.408216, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.93483167995468, + "seconds": 14.218142916, + "prefill_seconds": 0.1068725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.523246875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665833375, + "first": 3605395.189994625, + "last": 3605409.408132833, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.670543782090437, + "seconds": 14.430795291, + "prefill_seconds": 0.105029583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.31069625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665481, + "first": 3605394.977281125, + "last": 3605409.408325666, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.296077272876726, + "seconds": 13.215121208, + "prefill_seconds": 0.1150405, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.52606125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.666239916, + "first": 3605396.193254041, + "last": 3605409.408366625, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.419306216699344, + "seconds": 14.63892975, + "prefill_seconds": 0.102313917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.102579709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665622041, + "first": 3605394.76914725, + "last": 3605409.408009791, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.07181697629501, + "seconds": 14.110368666, + "prefill_seconds": 0.107833959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.631082958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.666305125, + "first": 3605395.297861458, + "last": 3605409.408104041, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.35592830677946, + "seconds": 13.891969708, + "prefill_seconds": 0.109609208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.849399459, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665948833, + "first": 3605395.516287375, + "last": 3605409.408301083, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.643012259790854, + "seconds": 12.981715667, + "prefill_seconds": 0.1172645, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.759228, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.66590175, + "first": 3605396.426690791, + "last": 3605409.408348791, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.212233857674743, + "seconds": 14.001577291, + "prefill_seconds": 0.108812375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.739878791, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665112375, + "first": 3605395.406704, + "last": 3605409.408167958, + "chunks": 222 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 134.84870970275804, + "ttft_max": 0.08884287532418966, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 134.8328003400667, + "seconds": 1.891231209, + "prefill_seconds": 0.088050667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.08823575, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 2 + }, + "speculative": { + "rounds": 86, + "drafted": 274, + "accepted": 169, + "acceptance_rate": 0.6167883211678832, + "tokens_per_round": 2.9767441860465116 + }, + "sent": 3605415.381412625, + "first": 3605415.4702555, + "last": 3605417.361263583, + "chunks": 87 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 232.29052564817403, + "ttft_max": 0.39637737488374114, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.44672236125179, + "seconds": 4.28955525, + "prefill_seconds": 0.101351875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.195082083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 48, + "accepted": 37, + "acceptance_rate": 0.7708333333333334, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605417.367488291, + "first": 3605417.563336, + "last": 3605421.85298975, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 62.36256925225751, + "seconds": 4.08899125, + "prefill_seconds": 0.100780375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.395675208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 48, + "accepted": 37, + "acceptance_rate": 0.7708333333333334, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605417.367552958, + "first": 3605417.763930333, + "last": 3605421.853057125, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.86286961475636, + "seconds": 4.189746583, + "prefill_seconds": 0.0998135, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.294881542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 48, + "accepted": 37, + "acceptance_rate": 0.7708333333333334, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605417.367618916, + "first": 3605417.6631545, + "last": 3605421.853040291, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.0744488389433, + "seconds": 4.390915542, + "prefill_seconds": 0.093563875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.093795125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 48, + "accepted": 37, + "acceptance_rate": 0.7708333333333334, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605417.367393208, + "first": 3605417.462004166, + "last": 3605421.852839708, + "chunks": 219 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 269.42968125685593, + "ttft_max": 0.7622620002366602, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.41528397005163, + "seconds": 7.002554208, + "prefill_seconds": 0.0957725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.665132083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858775541, + "first": 3605422.524708875, + "last": 3605429.527125458, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.53646371845895, + "seconds": 7.383500583, + "prefill_seconds": 0.093928333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.284276834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858546583, + "first": 3605422.143683916, + "last": 3605429.527139208, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.92196447130425, + "seconds": 6.906458084, + "prefill_seconds": 0.096173416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.761258875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858615583, + "first": 3605422.620877583, + "last": 3605429.527359791, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.98216209825066, + "seconds": 7.289429375, + "prefill_seconds": 0.094067792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.378250666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858727333, + "first": 3605422.237760875, + "last": 3605429.527248791, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.446830935158296, + "seconds": 7.193873, + "prefill_seconds": 0.09555475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.473801166, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858674, + "first": 3605422.333324208, + "last": 3605429.527215875, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.10270598285169, + "seconds": 7.4774125, + "prefill_seconds": 0.093924667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.190319, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858834375, + "first": 3605422.049752291, + "last": 3605429.527175708, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.92392179763727, + "seconds": 7.098334125, + "prefill_seconds": 0.095578625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.569345542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.859005666, + "first": 3605422.428939458, + "last": 3605429.527267541, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.67961369351017, + "seconds": 7.57134575, + "prefill_seconds": 0.096190333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.096446583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858450958, + "first": 3605421.955810916, + "last": 3605429.527082541, + "chunks": 228 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 307.8591382227815, + "ttft_max": 1.5631438749842346, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.962722485692893, + "seconds": 12.773808792, + "prefill_seconds": 0.096862583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.567743333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.533959458, + "first": 3605430.103254916, + "last": 3605442.877523375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.66563851479748, + "seconds": 12.966779584, + "prefill_seconds": 0.095515, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.3748605, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534009333, + "first": 3605429.910252666, + "last": 3605442.877322541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.379971505472856, + "seconds": 13.157914083, + "prefill_seconds": 0.093905833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.183785542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534398958, + "first": 3605429.719103583, + "last": 3605442.876903291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.10659876382601, + "seconds": 12.081529708, + "prefill_seconds": 0.100075125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.259563292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.5340755, + "first": 3605430.795726333, + "last": 3605442.877671583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.28345692918465, + "seconds": 11.981136375, + "prefill_seconds": 0.1004525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.359942458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534133125, + "first": 3605430.89622525, + "last": 3605442.877421458, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.933271174510256, + "seconds": 12.181564834, + "prefill_seconds": 0.099594042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.159546209, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534439041, + "first": 3605430.695691541, + "last": 3605442.877127708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.242652886598563, + "seconds": 13.251811042, + "prefill_seconds": 0.089549083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089800042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534185125, + "first": 3605429.625141666, + "last": 3605442.876929125, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.52187078626089, + "seconds": 13.062272709, + "prefill_seconds": 0.095650833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.279312375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534242083, + "first": 3605429.814729958, + "last": 3605442.877101833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.59748518639467, + "seconds": 12.380152125, + "prefill_seconds": 0.099545667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.961020708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534475958, + "first": 3605430.497009958, + "last": 3605442.877838291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.763545246040287, + "seconds": 12.281139708, + "prefill_seconds": 0.099029959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.060062583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534515041, + "first": 3605430.596032833, + "last": 3605442.877956, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.812497437850745, + "seconds": 12.870664125, + "prefill_seconds": 0.096127583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.470914666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534545208, + "first": 3605430.006379833, + "last": 3605442.876971416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.116567752228978, + "seconds": 12.676118667, + "prefill_seconds": 0.097671334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.665405875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.5342855, + "first": 3605430.200931208, + "last": 3605442.877004958, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.647095139435265, + "seconds": 11.779871542, + "prefill_seconds": 0.10088975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.561067417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534324333, + "first": 3605431.097468208, + "last": 3605442.877751208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.46325171461922, + "seconds": 11.880772, + "prefill_seconds": 0.100381917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.460275792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.5345785, + "first": 3605430.99657175, + "last": 3605442.8774815, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.43321193835248, + "seconds": 12.479682625, + "prefill_seconds": 0.098590583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.861649792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534357583, + "first": 3605430.397450916, + "last": 3605442.8771395, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.273075142073512, + "seconds": 12.5782595, + "prefill_seconds": 0.097901292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.763198375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.533873541, + "first": 3605430.29883175, + "last": 3605442.877249875, + "chunks": 256 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 145.3755917478066, + "ttft_max": 0.09841450024396181, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 145.36730782270743, + "seconds": 1.754177083, + "prefill_seconds": 0.097816084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098008292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 2 + }, + "speculative": { + "rounds": 78, + "drafted": 285, + "accepted": 177, + "acceptance_rate": 0.6210526315789474, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3605442.883334708, + "first": 3605442.981749208, + "last": 3605444.735826333, + "chunks": 79 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 218.4538282582019, + "ttft_max": 0.44721245765686035, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.43069322667104, + "seconds": 4.440134459, + "prefill_seconds": 0.11613075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.330156041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605444.741474708, + "first": 3605445.072298333, + "last": 3605449.512541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.61526033658863, + "seconds": 4.669024709, + "prefill_seconds": 0.101002, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101313875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605444.741541416, + "first": 3605444.843362125, + "last": 3605449.51230525, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 55.96697684991846, + "seconds": 4.55625825, + "prefill_seconds": 0.112759209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.214062, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605444.741598166, + "first": 3605444.9561535, + "last": 3605449.512485458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.97507051495962, + "seconds": 4.323860875, + "prefill_seconds": 0.116266625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.446445666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605444.741394, + "first": 3605445.188606458, + "last": 3605449.512434791, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 276.2903641691943, + "ttft_max": 0.8495748750865459, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.54092742950388, + "seconds": 7.382546416, + "prefill_seconds": 0.099146834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.099410292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518375708, + "first": 3605449.618810166, + "last": 3605457.001380083, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.823851195836035, + "seconds": 6.741777792, + "prefill_seconds": 0.108070917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.740279375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518509833, + "first": 3605450.259891375, + "last": 3605457.002291791, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.09481406821825, + "seconds": 7.064726792, + "prefill_seconds": 0.106603333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.417260542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518622416, + "first": 3605449.936744791, + "last": 3605457.002059708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.03749687564972, + "seconds": 7.277917167, + "prefill_seconds": 0.104644917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2040505, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518439166, + "first": 3605449.72347175, + "last": 3605457.002032125, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.649998499347845, + "seconds": 6.957708334, + "prefill_seconds": 0.107106167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.524294333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.51878775, + "first": 3605450.043845541, + "last": 3605457.002192833, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.441681761612095, + "seconds": 6.633424666, + "prefill_seconds": 0.108358833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.848550417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518678541, + "first": 3605450.368253416, + "last": 3605457.002250708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.227168873286004, + "seconds": 6.849835959, + "prefill_seconds": 0.107952625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.632268834, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518562333, + "first": 3605450.151802666, + "last": 3605457.002346708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.55849484955847, + "seconds": 7.171282167, + "prefill_seconds": 0.106634083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.310710417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.51828825, + "first": 3605449.830112083, + "last": 3605457.001886833, + "chunks": 196 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 305.7837592707918, + "ttft_max": 1.7033216250129044, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.227481228525594, + "seconds": 12.606611625, + "prefill_seconds": 0.107409542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.833712583, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.00861875, + "first": 3605457.844106125, + "last": 3605470.450826708, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.328889265817107, + "seconds": 11.955615542, + "prefill_seconds": 0.109789625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.484054917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008686833, + "first": 3605458.4952175, + "last": 3605470.450810958, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.567234730667614, + "seconds": 13.031989625, + "prefill_seconds": 0.104422084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.408458958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.00904975, + "first": 3605457.418656166, + "last": 3605470.450537833, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.134836567906717, + "seconds": 12.065387834, + "prefill_seconds": 0.108807208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.374314042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009264791, + "first": 3605458.385420666, + "last": 3605470.450794625, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.580029918216074, + "seconds": 12.390652541, + "prefill_seconds": 0.108638917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.0496385, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008748625, + "first": 3605458.060122333, + "last": 3605470.45078075, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.726250061702938, + "seconds": 12.926937416, + "prefill_seconds": 0.105059, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.513551125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008799458, + "first": 3605457.5237645, + "last": 3605470.450574833, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.111733770131554, + "seconds": 13.342588541, + "prefill_seconds": 0.098031875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098241875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009110625, + "first": 3605457.108064458, + "last": 3605470.4505095, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.411757826194172, + "seconds": 13.136368292, + "prefill_seconds": 0.103489208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.304175709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008959458, + "first": 3605457.314250791, + "last": 3605470.450658833, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.761268916543667, + "seconds": 12.28248625, + "prefill_seconds": 0.108184833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.15796075, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009216583, + "first": 3605458.168288208, + "last": 3605470.450773291, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.723004468900438, + "seconds": 11.738707708, + "prefill_seconds": 0.108524208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.700951291, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008847083, + "first": 3605458.712168708, + "last": 3605470.450819875, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.945919433429196, + "seconds": 12.174208958, + "prefill_seconds": 0.10830875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.265652833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009310833, + "first": 3605458.276619291, + "last": 3605470.450755083, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.889079449391613, + "seconds": 12.821106208, + "prefill_seconds": 0.105834375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.619311958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009351708, + "first": 3605457.629581875, + "last": 3605470.450625041, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.40119374239388, + "seconds": 12.499268583, + "prefill_seconds": 0.107347, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.940954208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.0090025, + "first": 3605457.951464666, + "last": 3605470.450725125, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.260036328797145, + "seconds": 13.239850416, + "prefill_seconds": 0.102721667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200804834, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008900416, + "first": 3605457.210753041, + "last": 3605470.450522791, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.056534521066187, + "seconds": 12.714060833, + "prefill_seconds": 0.107087416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.726305417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009167583, + "first": 3605457.736689583, + "last": 3605470.450684333, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.524041935164966, + "seconds": 11.847217208, + "prefill_seconds": 0.108408375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.592457708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008531416, + "first": 3605458.603612666, + "last": 3605470.450765791, + "chunks": 254 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 134.04015127905572, + "ttft_max": 0.08502129185944796, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 134.03280939207315, + "seconds": 1.90251925, + "prefill_seconds": 0.084423209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.084615291, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605475.610554166, + "first": 3605475.695575458, + "last": 3605477.5979905, + "chunks": 205 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 245.54320777721466, + "ttft_max": 0.34452699963003397, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 64.03927407833346, + "seconds": 3.981931458, + "prefill_seconds": 0.085632083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.25774, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605477.614962666, + "first": 3605477.873537583, + "last": 3605481.855391875, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.45556544127109, + "seconds": 3.895772625, + "prefill_seconds": 0.086155875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.343880375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605477.6151625, + "first": 3605477.9596895, + "last": 3605481.855459041, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 62.691247363564464, + "seconds": 4.067553458, + "prefill_seconds": 0.086513208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.172089333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605477.615059958, + "first": 3605477.787903041, + "last": 3605481.855448625, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.385409103510334, + "seconds": 4.154081625, + "prefill_seconds": 0.085394667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.085709541, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605477.614839208, + "first": 3605477.7014425, + "last": 3605481.855497541, + "chunks": 216 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 270.7423117196063, + "ttft_max": 0.7330167503096163, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.84593855412964, + "seconds": 7.534138833, + "prefill_seconds": 0.0900295, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.090248, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.861976458, + "first": 3605481.953038875, + "last": 3605489.487249041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.1149320635995, + "seconds": 7.261867958, + "prefill_seconds": 0.090984167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.362650291, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862190875, + "first": 3605482.225547375, + "last": 3605489.487622208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.562696398712745, + "seconds": 7.170434917, + "prefill_seconds": 0.091634833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.454281416, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862037, + "first": 3605482.317202208, + "last": 3605489.487771833, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.99559208154767, + "seconds": 6.892713041, + "prefill_seconds": 0.092998417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.732097667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862138625, + "first": 3605482.595155375, + "last": 3605489.487726291, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.68031094366792, + "seconds": 7.35287525, + "prefill_seconds": 0.090826291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.271782708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862229875, + "first": 3605482.134551708, + "last": 3605489.487819208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.02547021935962, + "seconds": 7.078325375, + "prefill_seconds": 0.092130791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.546408625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862090458, + "first": 3605482.409339833, + "last": 3605489.487566125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.503284092463026, + "seconds": 6.985672833, + "prefill_seconds": 0.092687833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.6390715, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862369958, + "first": 3605482.502132416, + "last": 3605489.48766875, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.258055273838764, + "seconds": 7.443504833, + "prefill_seconds": 0.090699459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.180952834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.861897625, + "first": 3605482.04376525, + "last": 3605489.487878875, + "chunks": 229 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 292.93412642405116, + "ttft_max": 1.4647241248749197, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.926920256904737, + "seconds": 13.472873375, + "prefill_seconds": 0.09103375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.546364292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494451375, + "first": 3605490.042471791, + "last": 3605503.515362625, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.733000571158392, + "seconds": 12.922515209, + "prefill_seconds": 0.091383459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.096374833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494896916, + "first": 3605490.59284475, + "last": 3605503.5152445, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.550024820973466, + "seconds": 13.746612334, + "prefill_seconds": 0.090700209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.272539583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494518625, + "first": 3605489.76867875, + "last": 3605503.515130208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.456901603825724, + "seconds": 13.105889375, + "prefill_seconds": 0.091986333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.91311975, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494632416, + "first": 3605490.409464791, + "last": 3605503.515424291, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.308820640498432, + "seconds": 13.927713041, + "prefill_seconds": 0.091075125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0913155, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.49495075, + "first": 3605489.587432416, + "last": 3605503.515098, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.594456981695206, + "seconds": 13.0138845, + "prefill_seconds": 0.091970959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.005073208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494578333, + "first": 3605490.501446208, + "last": 3605503.515478458, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.30949796549612, + "seconds": 12.555701792, + "prefill_seconds": 0.091528292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.462739375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494995833, + "first": 3605490.959719958, + "last": 3605503.515442125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.056723214752623, + "seconds": 13.38110425, + "prefill_seconds": 0.091652, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.637867125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.495036, + "first": 3605490.134127541, + "last": 3605503.5153105, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.67458408863247, + "seconds": 13.654922583, + "prefill_seconds": 0.091694166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.364163875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494694291, + "first": 3605489.860316083, + "last": 3605503.515465708, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.321338124166765, + "seconds": 13.197843667, + "prefill_seconds": 0.091791208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.821177041, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494736916, + "first": 3605490.3174595, + "last": 3605503.515433416, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.80005716768568, + "seconds": 13.563788542, + "prefill_seconds": 0.091140583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.455208083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.49508525, + "first": 3605489.951429166, + "last": 3605503.515194958, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.187885356109707, + "seconds": 13.289635375, + "prefill_seconds": 0.091527333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.729372166, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.49484925, + "first": 3605490.225666791, + "last": 3605503.515388125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.162431911887484, + "seconds": 12.647283875, + "prefill_seconds": 0.091754458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.3712595, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494791166, + "first": 3605490.868178291, + "last": 3605503.515450375, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.01728296135841, + "seconds": 12.738991625, + "prefill_seconds": 0.0919845, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.279713875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.495518708, + "first": 3605490.776427416, + "last": 3605503.515411041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.428504408437014, + "seconds": 13.837259625, + "prefill_seconds": 0.090444042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.181785917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.49517225, + "first": 3605489.677895625, + "last": 3605503.515071041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.87381691536153, + "seconds": 12.830952458, + "prefill_seconds": 0.091586458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.187832792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494358, + "first": 3605490.68442925, + "last": 3605503.515375041, + "chunks": 229 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 134.8330796716681, + "ttft_max": 0.13193579204380512, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 134.82593874550517, + "seconds": 1.891327458, + "prefill_seconds": 0.131301792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131517041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605503.525222083, + "first": 3605503.657157875, + "last": 3605505.548385166, + "chunks": 145 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 225.21937567407258, + "ttft_max": 0.5493802912533283, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.06139216612445, + "seconds": 4.391902958, + "prefill_seconds": 0.136901375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.27295775, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605505.554539708, + "first": 3605505.828181583, + "last": 3605510.220012125, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 56.306342560787606, + "seconds": 4.528797084, + "prefill_seconds": 0.135906542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.13614225, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605505.554665291, + "first": 3605505.69124725, + "last": 3605510.220045833, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.95063353421815, + "seconds": 4.116180666, + "prefill_seconds": 0.137669792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.548800833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605505.55460375, + "first": 3605506.103984041, + "last": 3605510.220164875, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.94556237694263, + "seconds": 4.2538595, + "prefill_seconds": 0.138106334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.411087208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605505.5544525, + "first": 3605505.96628925, + "last": 3605510.22013575, + "chunks": 167 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 259.820332969546, + "ttft_max": 0.902233749628067, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.54192665919949, + "seconds": 7.174625125, + "prefill_seconds": 0.114212292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.785898625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226858166, + "first": 3605511.013980875, + "last": 3605518.18909125, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.447547902386255, + "seconds": 7.402558833, + "prefill_seconds": 0.1132025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.558141875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226640083, + "first": 3605510.785963583, + "last": 3605518.188963833, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.985114146738646, + "seconds": 7.288814292, + "prefill_seconds": 0.1137675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.671693375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226808916, + "first": 3605510.899731583, + "last": 3605518.1890325, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.9289249370816, + "seconds": 7.515711166, + "prefill_seconds": 0.112677292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.445253042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.22669775, + "first": 3605510.672740375, + "last": 3605518.188532375, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.94573820483554, + "seconds": 7.739999584, + "prefill_seconds": 0.110556541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.220648625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226751083, + "first": 3605510.448116708, + "last": 3605518.188665958, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.12177692157362, + "seconds": 7.059453375, + "prefill_seconds": 0.115266209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.901208792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.227011333, + "first": 3605511.129245083, + "last": 3605518.188815666, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.481980621250365, + "seconds": 7.8505065, + "prefill_seconds": 0.109736375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.110060875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226935625, + "first": 3605510.33751175, + "last": 3605518.188263375, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.42890668679567, + "seconds": 7.628128625, + "prefill_seconds": 0.111950875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.332594958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226550291, + "first": 3605510.560035833, + "last": 3605518.188732541, + "chunks": 196 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 306.84608203546827, + "ttft_max": 1.7599479579366744, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.17835935085708, + "seconds": 13.296236416, + "prefill_seconds": 0.107219209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.107445625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195807791, + "first": 3605518.304832041, + "last": 3605531.601008625, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.832565639618313, + "seconds": 12.857640541, + "prefill_seconds": 0.110754666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.54585225, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195857416, + "first": 3605518.743459541, + "last": 3605531.601181958, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.897513191293374, + "seconds": 11.645157958, + "prefill_seconds": 0.108737208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.757228209, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196149458, + "first": 3605519.956097416, + "last": 3605531.601200083, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.36116420187307, + "seconds": 12.523841833, + "prefill_seconds": 0.111492834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.879644042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195917958, + "first": 3605519.077346291, + "last": 3605531.601344041, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.497621232863647, + "seconds": 13.0785185, + "prefill_seconds": 0.109325875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.325160334, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196680708, + "first": 3605518.522573291, + "last": 3605531.601083916, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.33601166000266, + "seconds": 13.187828208, + "prefill_seconds": 0.108400667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.215765959, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195975708, + "first": 3605518.413255375, + "last": 3605531.601117, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.694897027351384, + "seconds": 11.753916125, + "prefill_seconds": 0.108971333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.648525292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196330583, + "first": 3605519.847375333, + "last": 3605531.601384875, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.18146261034833, + "seconds": 12.63535775, + "prefill_seconds": 0.111334958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.76831725, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.1960735, + "first": 3605518.965816791, + "last": 3605531.601354958, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.299143038827552, + "seconds": 11.972312667, + "prefill_seconds": 0.109716708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.430249917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196623625, + "first": 3605519.628945291, + "last": 3605531.601218833, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.105705712451527, + "seconds": 12.082040917, + "prefill_seconds": 0.110033333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.320596875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196563083, + "first": 3605519.519230083, + "last": 3605531.601266041, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.91525583241108, + "seconds": 12.192057417, + "prefill_seconds": 0.1104875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.210580166, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196260583, + "first": 3605519.409184083, + "last": 3605531.601251291, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.663188221835757, + "seconds": 12.968395416, + "prefill_seconds": 0.110114292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.435185958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196021708, + "first": 3605518.632716416, + "last": 3605531.60104925, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.72746324258728, + "seconds": 12.3025185, + "prefill_seconds": 0.110307333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.100176625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.19639025, + "first": 3605519.298645125, + "last": 3605531.601400708, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.00525320846724, + "seconds": 12.746651959, + "prefill_seconds": 0.110993667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.656783625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196212041, + "first": 3605518.854473916, + "last": 3605531.601234666, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.495638724570185, + "seconds": 11.8628715, + "prefill_seconds": 0.109426, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.539636958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196869666, + "first": 3605519.738365708, + "last": 3605531.601364166, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.543276273335106, + "seconds": 12.412820458, + "prefill_seconds": 0.111021292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.990023, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195726041, + "first": 3605519.188370916, + "last": 3605531.6011585, + "chunks": 238 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 131.02762955084603, + "ttft_max": 0.09522395813837647, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 131.01969958356412, + "seconds": 1.946272208, + "prefill_seconds": 0.094651084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094815916, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605536.725807625, + "first": 3605536.821031583, + "last": 3605538.767186, + "chunks": 205 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 223.55292221131992, + "ttft_max": 0.41851866664364934, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.185937215559534, + "seconds": 4.45913825, + "prefill_seconds": 0.103478292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.205440625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605538.77615325, + "first": 3605538.982185041, + "last": 3605543.441316208, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.04708515180622, + "seconds": 4.246667417, + "prefill_seconds": 0.107146875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.417955625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605538.776206083, + "first": 3605539.19472475, + "last": 3605543.441306208, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 55.88888894368192, + "seconds": 4.56262425, + "prefill_seconds": 0.101644542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101953875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605538.77628525, + "first": 3605538.87870325, + "last": 3605543.441272791, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.56940889280687, + "seconds": 4.353808666, + "prefill_seconds": 0.105337417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.310781958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605538.77606675, + "first": 3605539.087549666, + "last": 3605543.44138125, + "chunks": 216 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 267.110955393874, + "ttft_max": 0.7607742501422763, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.088886715693185, + "seconds": 7.065887125, + "prefill_seconds": 0.096627542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.663958834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448670208, + "first": 3605544.11365275, + "last": 3605551.179667666, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.120935183051415, + "seconds": 7.260626708, + "prefill_seconds": 0.095467708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.469219375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448989833, + "first": 3605543.918952083, + "last": 3605551.179549, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.80224652784556, + "seconds": 7.543877292, + "prefill_seconds": 0.093102875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.185995834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448920625, + "first": 3605543.635653291, + "last": 3605551.1797455, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.39017811252395, + "seconds": 7.636976333, + "prefill_seconds": 0.092668541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.092922833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448741083, + "first": 3605543.542469833, + "last": 3605551.179358875, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.60209569225996, + "seconds": 7.162499708, + "prefill_seconds": 0.098108875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.567303125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448808416, + "first": 3605544.017031583, + "last": 3605551.179516375, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.66521645381622, + "seconds": 7.356076958, + "prefill_seconds": 0.094272583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.373741791, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448862708, + "first": 3605543.823425708, + "last": 3605551.1794025, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.586875211863024, + "seconds": 6.969712459, + "prefill_seconds": 0.096179833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.760045167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.44908225, + "first": 3605544.2098565, + "last": 3605551.179606166, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.22654626729075, + "seconds": 7.450357334, + "prefill_seconds": 0.093555, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.279560833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448587083, + "first": 3605543.729159208, + "last": 3605551.179385541, + "chunks": 229 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 304.5591714806901, + "ttft_max": 1.5759439589455724, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.896696675450308, + "seconds": 12.816197792, + "prefill_seconds": 0.098274708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.673495459, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187829666, + "first": 3605551.862587583, + "last": 3605564.679362333, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.454755059424507, + "seconds": 13.107335416, + "prefill_seconds": 0.096827833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.381084167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187347875, + "first": 3605551.5699955, + "last": 3605564.678450041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.173570848218482, + "seconds": 13.299557084, + "prefill_seconds": 0.094394833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188357292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187417791, + "first": 3605551.377271083, + "last": 3605564.676923666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.401400417421726, + "seconds": 11.915108125, + "prefill_seconds": 0.100877667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.57352325, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187671166, + "first": 3605552.763615125, + "last": 3605564.679279708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.59868609461536, + "seconds": 13.011076292, + "prefill_seconds": 0.097481084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.478319084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187617791, + "first": 3605551.667259833, + "last": 3605564.678642791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.701212860267265, + "seconds": 12.318118833, + "prefill_seconds": 0.100245917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.170799667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.18805875, + "first": 3605552.3602425, + "last": 3605564.679030583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.745475464432126, + "seconds": 12.914350959, + "prefill_seconds": 0.097031708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.575222417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187489, + "first": 3605551.764302416, + "last": 3605564.678928875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.312291263568483, + "seconds": 13.204026209, + "prefill_seconds": 0.095629917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.284033292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187895708, + "first": 3605551.472959958, + "last": 3605564.676970541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.044453199714567, + "seconds": 12.117207208, + "prefill_seconds": 0.100306625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.371618125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187939375, + "first": 3605552.5613415, + "last": 3605564.678822625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.53412080141582, + "seconds": 12.418354916, + "prefill_seconds": 0.099302875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.070665625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187554916, + "first": 3605552.259993458, + "last": 3605564.678601375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.053464962880685, + "seconds": 12.716006958, + "prefill_seconds": 0.0985685, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.772066042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187994291, + "first": 3605551.96115275, + "last": 3605564.677346375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.87184934034128, + "seconds": 12.217412834, + "prefill_seconds": 0.100775084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.271502834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.188120791, + "first": 3605552.4610205, + "last": 3605564.679188333, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.211309971776885, + "seconds": 12.616698292, + "prefill_seconds": 0.099337709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.871298708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.188180583, + "first": 3605552.06051425, + "last": 3605564.677739375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.03846760643467, + "seconds": 13.393935125, + "prefill_seconds": 0.093820291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094052667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187727583, + "first": 3605551.282950791, + "last": 3605564.678359958, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.22169114824537, + "seconds": 12.0160075, + "prefill_seconds": 0.101354042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.472934666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187773083, + "first": 3605552.6626945, + "last": 3605564.67910925, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.37291427891917, + "seconds": 12.516618708, + "prefill_seconds": 0.100147458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.971464084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187247375, + "first": 3605552.160674291, + "last": 3605564.67800675, + "chunks": 256 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 154.95886207861588, + "ttft_max": 0.10082387505099177, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 154.94833976171662, + "seconds": 1.645709792, + "prefill_seconds": 0.099591459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.100275959, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605564.688457666, + "first": 3605564.789281541, + "last": 3605566.434879583, + "chunks": 145 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 242.36047700354084, + "ttft_max": 0.42187445890158415, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.59083789787342, + "seconds": 4.208557083, + "prefill_seconds": 0.102118834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.102311542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605566.441818333, + "first": 3605566.544721125, + "last": 3605570.753210666, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.55965406317821, + "seconds": 3.889587333, + "prefill_seconds": 0.1074365, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.421266333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605566.441888166, + "first": 3605566.863762625, + "last": 3605570.753328125, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 62.141115395800036, + "seconds": 4.103563291, + "prefill_seconds": 0.105009958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2073145, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605566.441946958, + "first": 3605566.649737666, + "last": 3605570.753248791, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.79769276269591, + "seconds": 3.99700975, + "prefill_seconds": 0.106556875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.313817375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605566.441734333, + "first": 3605566.756301208, + "last": 3605570.753289875, + "chunks": 167 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 263.8200654034127, + "ttft_max": 0.9031712496653199, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.962132787976955, + "seconds": 7.508362375, + "prefill_seconds": 0.112015833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.335270667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760726458, + "first": 3605571.09699175, + "last": 3605578.605317541, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.573171744850306, + "seconds": 7.168323416, + "prefill_seconds": 0.114088625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.675248375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760784666, + "first": 3605571.437124833, + "last": 3605578.605774666, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.01562509172935, + "seconds": 7.282463167, + "prefill_seconds": 0.112924792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.561342583, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760840583, + "first": 3605571.323135333, + "last": 3605578.605375083, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.14625113408863, + "seconds": 7.0546735, + "prefill_seconds": 0.113693125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.788898458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.761060916, + "first": 3605571.550822083, + "last": 3605578.605628708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.481382512777, + "seconds": 7.395295125, + "prefill_seconds": 0.11306225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.448364625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.7608885, + "first": 3605571.210063208, + "last": 3605578.605350541, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.73751667190106, + "seconds": 6.941133291, + "prefill_seconds": 0.113508459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.902484542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.761161208, + "first": 3605571.664332458, + "last": 3605578.605549208, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.97946084302359, + "seconds": 7.732085167, + "prefill_seconds": 0.111460375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.111665417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760939416, + "first": 3605570.873231666, + "last": 3605578.605298208, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.462928365466354, + "seconds": 7.620373125, + "prefill_seconds": 0.111713334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.22331875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760643791, + "first": 3605570.98495875, + "last": 3605578.605269208, + "chunks": 196 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 310.3552343905495, + "ttft_max": 1.649341749958694, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.601231295597064, + "seconds": 11.804882625, + "prefill_seconds": 0.103076792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.438742334, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613517666, + "first": 3605580.054471916, + "last": 3605591.860033416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.514480305695376, + "seconds": 12.430244208, + "prefill_seconds": 0.106008791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.813621208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.6139825, + "first": 3605579.428936916, + "last": 3605591.859972666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.17141942444813, + "seconds": 12.641648792, + "prefill_seconds": 0.103951, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.602347542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613593041, + "first": 3605579.217488416, + "last": 3605591.859112083, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.69574863463283, + "seconds": 12.946956459, + "prefill_seconds": 0.0992075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.297069625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613781833, + "first": 3605578.912178, + "last": 3605591.860200125, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.41421071156215, + "seconds": 11.907980333, + "prefill_seconds": 0.104237667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.3356565, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.6137215, + "first": 3605579.951374291, + "last": 3605591.86023075, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.045202774397332, + "seconds": 12.116775625, + "prefill_seconds": 0.104969083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.12696575, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614247583, + "first": 3605579.742488625, + "last": 3605591.86000325, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.688762015476815, + "seconds": 12.325532084, + "prefill_seconds": 0.104711167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.918210667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613651375, + "first": 3605579.533632958, + "last": 3605591.859162416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.795983017424774, + "seconds": 11.6994035, + "prefill_seconds": 0.105472208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.544115209, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614026916, + "first": 3605580.159944208, + "last": 3605591.859847833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.341035948504953, + "seconds": 12.536234666, + "prefill_seconds": 0.10541625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.707677917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613929291, + "first": 3605579.322897208, + "last": 3605591.859859708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.398889956353763, + "seconds": 13.145082042, + "prefill_seconds": 0.098952833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.099173125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613878041, + "first": 3605578.714004916, + "last": 3605591.859769666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.228452308918147, + "seconds": 12.012180459, + "prefill_seconds": 0.104630334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.231449042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614070458, + "first": 3605579.847168583, + "last": 3605591.859674458, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.849658943156147, + "seconds": 12.846568333, + "prefill_seconds": 0.100359583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.397427875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613832041, + "first": 3605579.012549083, + "last": 3605591.860064916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.990573014412327, + "seconds": 11.595877917, + "prefill_seconds": 0.103537833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.647648167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614167625, + "first": 3605580.263509375, + "last": 3605591.859724208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.864530693861955, + "seconds": 12.221698333, + "prefill_seconds": 0.103841959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.022063, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614118708, + "first": 3605579.637514458, + "last": 3605591.859193, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.006909837448564, + "seconds": 12.7455965, + "prefill_seconds": 0.100966709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.498359167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614321291, + "first": 3605579.113519083, + "last": 3605591.859611208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.54596923465485, + "seconds": 13.046168084, + "prefill_seconds": 0.098901375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.197920209, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613433208, + "first": 3605578.812915958, + "last": 3605591.859001875, + "chunks": 256 + } + ] + } + ], + "performance-qwen": [ + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 30.344307936698552, + "ttft_max": 0.23980883276090026, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 30.344048558503754, + "seconds": 8.403624833, + "prefill_seconds": 0.239020458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.239324791, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 335, + "accepted": 200, + "acceptance_rate": 0.5970149253731343, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3604496.127048333, + "first": 3604496.366857166, + "last": 3604504.770410166, + "chunks": 56 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 30.93870088644765, + "ttft_max": 25.042438166681677, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.830604264701808, + "seconds": 8.011158, + "prefill_seconds": 0.229463833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 8.644388292, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 329, + "accepted": 202, + "acceptance_rate": 0.6139817629179332, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604504.77142875, + "first": 3604513.416526083, + "last": 3604521.427605083, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.158479446950714, + "seconds": 8.183968041, + "prefill_seconds": 0.230723791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.231085125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 327, + "accepted": 200, + "acceptance_rate": 0.6116207951070336, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3604504.771560125, + "first": 3604505.003225333, + "last": 3604513.187041583, + "chunks": 56 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 32.17277731613354, + "seconds": 7.925955459, + "prefill_seconds": 0.23254725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 16.888045542, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 323, + "accepted": 202, + "acceptance_rate": 0.6253869969040248, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604504.771489125, + "first": 3604521.660191041, + "last": 3604529.586075875, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.25806670338704, + "seconds": 8.157894166, + "prefill_seconds": 0.227744292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 25.041650917, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 54, + "drafted": 328, + "accepted": 201, + "acceptance_rate": 0.6128048780487805, + "tokens_per_round": 4.7407407407407405 + }, + "sent": 3604504.771343333, + "first": 3604529.8137815, + "last": 3604537.971642583, + "chunks": 55 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.065841730942, + "ttft_max": 0.43003095779567957, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.06554077318274, + "seconds": 11.055452916, + "prefill_seconds": 0.429262125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.429609584, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 82, + "drafted": 345, + "accepted": 173, + "acceptance_rate": 0.5014492753623189, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3604537.973906, + "first": 3604538.403936958, + "last": 3604549.459245625, + "chunks": 83 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 22.55082188387638, + "ttft_max": 34.96590099995956, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 22.962253892898396, + "seconds": 11.105181625, + "prefill_seconds": 0.425930625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 11.91523375, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 83, + "drafted": 356, + "accepted": 172, + "acceptance_rate": 0.48314606741573035, + "tokens_per_round": 3.0843373493975905 + }, + "sent": 3604549.460459541, + "first": 3604561.376253166, + "last": 3604572.48135725, + "chunks": 84 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.04879780747181, + "seconds": 11.06348375, + "prefill_seconds": 0.425502417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.425864083, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 83, + "drafted": 356, + "accepted": 172, + "acceptance_rate": 0.48314606741573035, + "tokens_per_round": 3.0843373493975905 + }, + "sent": 3604549.460593666, + "first": 3604549.88709, + "last": 3604560.950315375, + "chunks": 84 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.037278457924852, + "seconds": 11.069015833, + "prefill_seconds": 0.430014417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 23.45031175, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 81, + "drafted": 360, + "accepted": 174, + "acceptance_rate": 0.48333333333333334, + "tokens_per_round": 3.1604938271604937 + }, + "sent": 3604549.4605245, + "first": 3604572.911377833, + "last": 3604583.980308791, + "chunks": 82 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.84952037711046, + "seconds": 10.692038916, + "prefill_seconds": 0.445968, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 34.965197042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 81, + "drafted": 348, + "accepted": 174, + "acceptance_rate": 0.5, + "tokens_per_round": 3.1604938271604937 + }, + "sent": 3604549.460381583, + "first": 3604584.426282583, + "last": 3604595.118257416, + "chunks": 82 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 31.3400196637318, + "ttft_max": 0.22861679224297404, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.33964026993998, + "seconds": 8.136660083, + "prefill_seconds": 0.227769375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228073333, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 375, + "accepted": 202, + "acceptance_rate": 0.5386666666666666, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604649.728608666, + "first": 3604649.957225458, + "last": 3604658.093787041, + "chunks": 54 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 48.70154286002622, + "ttft_max": 0.8868492497131228, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.442917404598337, + "seconds": 20.493586167, + "prefill_seconds": 0.660301083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.66058425, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 67, + "drafted": 217, + "accepted": 188, + "acceptance_rate": 0.8663594470046083, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604658.0960795, + "first": 3604658.757237458, + "last": 3604679.250707458, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.581526389590532, + "seconds": 20.267811083, + "prefill_seconds": 0.886135833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.886447709, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 67, + "drafted": 217, + "accepted": 188, + "acceptance_rate": 0.8663594470046083, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604658.096224625, + "first": 3604658.983073875, + "last": 3604679.250772125, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.308087453809105, + "seconds": 20.718084833, + "prefill_seconds": 0.435774209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.436020416, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 67, + "drafted": 218, + "accepted": 188, + "acceptance_rate": 0.8623853211009175, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604658.096145875, + "first": 3604658.532716958, + "last": 3604679.2507495, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.175402937052157, + "seconds": 20.943865375, + "prefill_seconds": 0.209930417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.210243625, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 67, + "drafted": 220, + "accepted": 188, + "acceptance_rate": 0.8545454545454545, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604658.095991625, + "first": 3604658.306877125, + "last": 3604679.250664625, + "chunks": 68 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 24.00490857949334, + "ttft_max": 0.4215462077409029, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 24.004729686343946, + "seconds": 10.622906541, + "prefill_seconds": 0.420831, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.421155459, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 462, + "accepted": 180, + "acceptance_rate": 0.38961038961038963, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3604679.252754833, + "first": 3604679.674301041, + "last": 3604690.297128416, + "chunks": 76 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 41.78288826705579, + "ttft_max": 1.6581098344177008, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.628013650766608, + "seconds": 23.993194625, + "prefill_seconds": 0.836824834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.837248625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 220, + "accepted": 158, + "acceptance_rate": 0.7181818181818181, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604690.2980735, + "first": 3604691.1358885, + "last": 3604715.129133166, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.004252625792686, + "seconds": 23.1728595, + "prefill_seconds": 1.238961458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.657580292, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 216, + "accepted": 158, + "acceptance_rate": 0.7314814814814815, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604690.298201916, + "first": 3604691.95631175, + "last": 3604715.129071458, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.445826937596213, + "seconds": 24.411662334, + "prefill_seconds": 0.418261625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.418658166, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 220, + "accepted": 158, + "acceptance_rate": 0.7181818181818181, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604690.298135, + "first": 3604690.717330833, + "last": 3604715.128903916, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.813981575196546, + "seconds": 23.580583916, + "prefill_seconds": 0.831247542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.249867292, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 216, + "accepted": 158, + "acceptance_rate": 0.7314814814814815, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604690.297998291, + "first": 3604691.548578375, + "last": 3604715.12923825, + "chunks": 98 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 36.11988572345143, + "ttft_max": 0.19940866669639945, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 36.11952098561528, + "seconds": 7.059894291, + "prefill_seconds": 0.198697, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198979292, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 54, + "drafted": 355, + "accepted": 201, + "acceptance_rate": 0.5661971830985916, + "tokens_per_round": 4.7407407407407405 + }, + "sent": 3604725.818032958, + "first": 3604726.017441625, + "last": 3604733.077264625, + "chunks": 55 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 32.39324544693206, + "ttft_max": 20.72907174983993, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 29.33421034201612, + "seconds": 8.692921917, + "prefill_seconds": 0.205756625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.20602725, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 375, + "accepted": 202, + "acceptance_rate": 0.5386666666666666, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604733.078777125, + "first": 3604733.28554225, + "last": 3604741.978387208, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 24.466282928760503, + "seconds": 10.422506792, + "prefill_seconds": 0.24281125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 17.766874792, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 62, + "drafted": 262, + "accepted": 193, + "acceptance_rate": 0.7366412213740458, + "tokens_per_round": 4.129032258064516 + }, + "sent": 3604733.0789195, + "first": 3604750.846356583, + "last": 3604761.2687955, + "chunks": 63 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.25422850288127, + "seconds": 10.965747583, + "prefill_seconds": 0.251704791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 20.728427125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 67, + "drafted": 277, + "accepted": 188, + "acceptance_rate": 0.6787003610108303, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604733.078850041, + "first": 3604753.807921791, + "last": 3604764.773588166, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 30.390343707071754, + "seconds": 8.390823166, + "prefill_seconds": 0.234339208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 9.133264833, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 368, + "accepted": 202, + "acceptance_rate": 0.5489130434782609, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604733.078688416, + "first": 3604742.212719333, + "last": 3604750.603482375, + "chunks": 54 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 22.499851186148447, + "ttft_max": 0.4421482081525028, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 22.499807345399603, + "seconds": 11.333430375, + "prefill_seconds": 0.441396833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.44167975, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 460, + "accepted": 180, + "acceptance_rate": 0.391304347826087, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3604764.775536208, + "first": 3604765.217684416, + "last": 3604776.551092708, + "chunks": 76 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 39.84032356389979, + "ttft_max": 1.768473374657333, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.13453022532727, + "seconds": 25.16150175, + "prefill_seconds": 0.885118083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.885429667, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 220, + "accepted": 159, + "acceptance_rate": 0.7227272727272728, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3604776.552390458, + "first": 3604777.438389916, + "last": 3604802.599801291, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.3173151293953, + "seconds": 24.715732417, + "prefill_seconds": 0.886504542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.331123084, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 218, + "accepted": 159, + "acceptance_rate": 0.7293577981651376, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3604776.552447875, + "first": 3604777.8841745, + "last": 3604802.599754083, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.960122290447101, + "seconds": 25.602095292, + "prefill_seconds": 0.444373291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.444644625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 220, + "accepted": 159, + "acceptance_rate": 0.7227272727272728, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3604776.55252475, + "first": 3604776.997629208, + "last": 3604802.599635708, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.502846912495857, + "seconds": 24.279131375, + "prefill_seconds": 1.323141792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.767754791, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 217, + "accepted": 159, + "acceptance_rate": 0.7327188940092166, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3604776.552310333, + "first": 3604778.320783708, + "last": 3604802.599830916, + "chunks": 97 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 36.20655991593265, + "ttft_max": 0.19942687498405576, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 36.20599550053157, + "seconds": 7.043032417, + "prefill_seconds": 0.198634458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198939541, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 310, + "accepted": 200, + "acceptance_rate": 0.6451612903225806, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3604812.635286291, + "first": 3604812.834713166, + "last": 3604819.877635791, + "chunks": 56 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 47.047737556009366, + "ttft_max": 0.841904915869236, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.294832987952992, + "seconds": 20.74042, + "prefill_seconds": 0.411415292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.411670375, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 78, + "drafted": 210, + "accepted": 177, + "acceptance_rate": 0.8428571428571429, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3604819.879005583, + "first": 3604820.291204, + "last": 3604841.03153175, + "chunks": 79 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.448224614983888, + "seconds": 20.484848875, + "prefill_seconds": 0.640674458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.841417875, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 79, + "drafted": 213, + "accepted": 176, + "acceptance_rate": 0.8262910798122066, + "tokens_per_round": 3.240506329113924 + }, + "sent": 3604819.879153375, + "first": 3604820.721058291, + "last": 3604841.205771791, + "chunks": 80 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.996807508383107, + "seconds": 21.255654875, + "prefill_seconds": 0.625017334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.625289167, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 83, + "drafted": 219, + "accepted": 172, + "acceptance_rate": 0.7853881278538812, + "tokens_per_round": 3.0843373493975905 + }, + "sent": 3604819.879072541, + "first": 3604820.504795458, + "last": 3604841.760405958, + "chunks": 84 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.17110093138091, + "seconds": 20.951268208, + "prefill_seconds": 0.200510875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200836458, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 78, + "drafted": 212, + "accepted": 177, + "acceptance_rate": 0.8349056603773585, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3604819.878924333, + "first": 3604820.080298625, + "last": 3604841.031495291, + "chunks": 79 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 21.34347493314762, + "ttft_max": 0.4058342077769339, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 21.34320473816755, + "seconds": 11.947596583, + "prefill_seconds": 0.405067875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.405350209, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 83, + "drafted": 360, + "accepted": 172, + "acceptance_rate": 0.4777777777777778, + "tokens_per_round": 3.0843373493975905 + }, + "sent": 3604841.762581833, + "first": 3604842.168416041, + "last": 3604854.115861375, + "chunks": 84 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 41.72305834262606, + "ttft_max": 1.741847958881408, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.618586816004251, + "seconds": 24.014495, + "prefill_seconds": 0.4395565, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.439896125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 95, + "drafted": 231, + "accepted": 160, + "acceptance_rate": 0.6926406926406926, + "tokens_per_round": 2.694736842105263 + }, + "sent": 3604854.117007833, + "first": 3604854.557580416, + "last": 3604878.571969666, + "chunks": 96 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.85651813294774, + "seconds": 23.488193625, + "prefill_seconds": 0.872232791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.312090042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 217, + "accepted": 158, + "acceptance_rate": 0.728110599078341, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604854.117268125, + "first": 3604855.429762958, + "last": 3604878.917895041, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.65955182687684, + "seconds": 23.9222065, + "prefill_seconds": 0.877797084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.878160459, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 227, + "accepted": 158, + "acceptance_rate": 0.6960352422907489, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604854.117089583, + "first": 3604854.995781625, + "last": 3604878.917951916, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.017110258099947, + "seconds": 23.145815375, + "prefill_seconds": 1.301217375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.741100167, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 98, + "drafted": 212, + "accepted": 157, + "acceptance_rate": 0.7405660377358491, + "tokens_per_round": 2.6122448979591835 + }, + "sent": 3604854.116930541, + "first": 3604855.8587785, + "last": 3604879.004493958, + "chunks": 99 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 36.12399860205154, + "ttft_max": 0.20252579217776656, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 36.123212024726776, + "seconds": 7.059172917, + "prefill_seconds": 0.200018875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200550916, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 310, + "accepted": 200, + "acceptance_rate": 0.6451612903225806, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3604889.219564666, + "first": 3604889.422090458, + "last": 3604896.481109666, + "chunks": 56 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 49.44176860435623, + "ttft_max": 0.8374648750759661, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.837368778818945, + "seconds": 19.863883666, + "prefill_seconds": 0.409651375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.409944125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 82, + "drafted": 216, + "accepted": 173, + "acceptance_rate": 0.8009259259259259, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3604896.482677875, + "first": 3604896.893221833, + "last": 3604916.757026166, + "chunks": 83 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.754208356907291, + "seconds": 19.993400834, + "prefill_seconds": 0.636964958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.836840709, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 84, + "drafted": 216, + "accepted": 171, + "acceptance_rate": 0.7916666666666666, + "tokens_per_round": 3.0476190476190474 + }, + "sent": 3604896.482735666, + "first": 3604897.320200541, + "last": 3604917.313493791, + "chunks": 85 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.619169315959981, + "seconds": 20.207352292, + "prefill_seconds": 0.622653792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.622908709, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 84, + "drafted": 217, + "accepted": 171, + "acceptance_rate": 0.7880184331797235, + "tokens_per_round": 3.0476190476190474 + }, + "sent": 3604896.482804208, + "first": 3604897.106206625, + "last": 3604917.313513666, + "chunks": 85 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.894398186944702, + "seconds": 19.776029583, + "prefill_seconds": 0.199632, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.199933875, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 81, + "drafted": 214, + "accepted": 174, + "acceptance_rate": 0.8130841121495327, + "tokens_per_round": 3.1604938271604937 + }, + "sent": 3604896.4826005, + "first": 3604896.683183708, + "last": 3604916.45914225, + "chunks": 82 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.096668462332214, + "ttft_max": 0.4583138329908252, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.096494306274323, + "seconds": 11.040636584, + "prefill_seconds": 0.457591875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.457907792, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 84, + "drafted": 338, + "accepted": 171, + "acceptance_rate": 0.5059171597633136, + "tokens_per_round": 3.0476190476190474 + }, + "sent": 3604917.315005208, + "first": 3604917.773319041, + "last": 3604928.813872375, + "chunks": 85 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 39.58191601199035, + "ttft_max": 1.6596389156766236, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.31685269722128, + "seconds": 24.71684025, + "prefill_seconds": 0.831686959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.248863583, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 99, + "drafted": 219, + "accepted": 156, + "acceptance_rate": 0.7123287671232876, + "tokens_per_round": 2.585858585858586 + }, + "sent": 3604928.815203625, + "first": 3604930.064585041, + "last": 3604954.781297375, + "chunks": 100 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.39658399943459, + "seconds": 24.527287041, + "prefill_seconds": 1.242135334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.659193041, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 101, + "drafted": 214, + "accepted": 154, + "acceptance_rate": 0.719626168224299, + "tokens_per_round": 2.5346534653465347 + }, + "sent": 3604928.81539575, + "first": 3604930.475034666, + "last": 3604955.00224225, + "chunks": 102 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.146833534859063, + "seconds": 25.13099275, + "prefill_seconds": 0.83436225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.834691042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 99, + "drafted": 232, + "accepted": 156, + "acceptance_rate": 0.6724137931034483, + "tokens_per_round": 2.585858585858586 + }, + "sent": 3604928.815273833, + "first": 3604929.650407416, + "last": 3604954.781352958, + "chunks": 100 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.981063525080605, + "seconds": 25.548379625, + "prefill_seconds": 0.416843875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.417150625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 99, + "drafted": 235, + "accepted": 156, + "acceptance_rate": 0.6638297872340425, + "tokens_per_round": 2.585858585858586 + }, + "sent": 3604928.815126666, + "first": 3604929.2328985, + "last": 3604954.781213166, + "chunks": 100 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 30.31858681923586, + "ttft_max": 0.19988291710615158, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 30.318100784216046, + "seconds": 8.410817083, + "prefill_seconds": 0.19909825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.199392041, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 52, + "drafted": 376, + "accepted": 203, + "acceptance_rate": 0.5398936170212766, + "tokens_per_round": 4.923076923076923 + }, + "sent": 3604965.120085958, + "first": 3604965.319968875, + "last": 3604973.730651125, + "chunks": 53 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 42.716145327090175, + "ttft_max": 1.1741919172927737, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.813385107043793, + "seconds": 23.581884625, + "prefill_seconds": 0.575820834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.57606975, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 242, + "accepted": 190, + "acceptance_rate": 0.7851239669421488, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3604973.732724208, + "first": 3604974.309460291, + "last": 3604997.891232416, + "chunks": 66 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.952668570562086, + "seconds": 23.281997292, + "prefill_seconds": 0.875755958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.876025834, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 239, + "accepted": 190, + "acceptance_rate": 0.7949790794979079, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3604973.732836041, + "first": 3604974.609402333, + "last": 3604997.891275625, + "chunks": 66 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.094450179357176, + "seconds": 22.984464834, + "prefill_seconds": 0.894214334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.1735915, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 239, + "accepted": 190, + "acceptance_rate": 0.7949790794979079, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3604973.732783083, + "first": 3604974.906975, + "last": 3604997.891289791, + "chunks": 66 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.679046785710884, + "seconds": 23.878535708, + "prefill_seconds": 0.27908075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.279380125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 244, + "accepted": 190, + "acceptance_rate": 0.7786885245901639, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3604973.732628458, + "first": 3604974.012730708, + "last": 3604997.891204208, + "chunks": 66 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.72393278235333, + "ttft_max": 0.3962868331000209, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.72372098737278, + "seconds": 10.748735417, + "prefill_seconds": 0.395573708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.395857292, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 469, + "accepted": 180, + "acceptance_rate": 0.3837953091684435, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3604997.893435625, + "first": 3604998.289722458, + "last": 3605009.038361916, + "chunks": 76 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 37.63360072289366, + "ttft_max": 1.6586603340692818, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.523506182264372, + "seconds": 26.775852834, + "prefill_seconds": 0.412750042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.413028125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 95, + "drafted": 226, + "accepted": 160, + "acceptance_rate": 0.7079646017699115, + "tokens_per_round": 2.694736842105263 + }, + "sent": 3605009.039835666, + "first": 3605009.453469416, + "last": 3605036.229273375, + "chunks": 96 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.861389867154255, + "seconds": 25.858423958, + "prefill_seconds": 1.24508075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.658040666, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 219, + "accepted": 158, + "acceptance_rate": 0.7214611872146118, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3605009.039891291, + "first": 3605010.698551625, + "last": 3605036.556908375, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.674276576325884, + "seconds": 26.358560042, + "prefill_seconds": 0.830139, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.830437917, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 95, + "drafted": 225, + "accepted": 160, + "acceptance_rate": 0.7111111111111111, + "tokens_per_round": 2.694736842105263 + }, + "sent": 3605009.039945166, + "first": 3605009.870907166, + "last": 3605036.229306958, + "chunks": 96 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.70758935738823, + "seconds": 26.268107417, + "prefill_seconds": 1.248113042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.248405541, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 219, + "accepted": 158, + "acceptance_rate": 0.7214611872146118, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3605009.039748541, + "first": 3605010.288857416, + "last": 3605036.556876958, + "chunks": 98 + } + ] + } + ], + "performance-qwen-fresh-cost-records": [ + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 36.15852812983743, + "ttft_max": 0.19951058318838477, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 36.15814210107307, + "seconds": 7.0523535, + "prefill_seconds": 0.198717, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198999834, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 54, + "drafted": 355, + "accepted": 201, + "acceptance_rate": 0.5661971830985916, + "tokens_per_round": 4.7407407407407405 + }, + "sent": 3606027.273496208, + "first": 3606027.473006791, + "last": 3606034.525285, + "chunks": 55 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 51.576538557442284, + "ttft_max": 0.8468245826661587, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.894425707530571, + "seconds": 19.775987375, + "prefill_seconds": 0.199771833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200094417, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 82, + "drafted": 210, + "accepted": 173, + "acceptance_rate": 0.8238095238095238, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3606034.526158833, + "first": 3606034.726779708, + "last": 3606054.502661083, + "chunks": 83 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 13.32983899880677, + "seconds": 19.1300135, + "prefill_seconds": 0.846160167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.846441083, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 82, + "drafted": 210, + "accepted": 173, + "acceptance_rate": 0.8238095238095238, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3606034.526308833, + "first": 3606035.373133416, + "last": 3606054.5032135, + "chunks": 83 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 13.178562543878034, + "seconds": 19.349606541, + "prefill_seconds": 0.626363083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.626663041, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 82, + "drafted": 210, + "accepted": 173, + "acceptance_rate": 0.8238095238095238, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3606034.526230625, + "first": 3606035.153333916, + "last": 3606054.503172208, + "chunks": 83 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 13.034482495107342, + "seconds": 19.563492459, + "prefill_seconds": 0.412400709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.412706292, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 82, + "drafted": 210, + "accepted": 173, + "acceptance_rate": 0.8238095238095238, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3606034.526077, + "first": 3606034.93936725, + "last": 3606054.503073791, + "chunks": 83 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 26.04380517925345, + "ttft_max": 0.3833406246267259, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 26.043570663924534, + "seconds": 9.791284125, + "prefill_seconds": 0.382626125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.382920334, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 448, + "accepted": 180, + "acceptance_rate": 0.4017857142857143, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606054.50418025, + "first": 3606054.887520875, + "last": 3606064.678716833, + "chunks": 76 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 25.201173385079827, + "ttft_max": 31.02874766662717, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 25.97921266330204, + "seconds": 9.815539959, + "prefill_seconds": 0.372683292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 20.837619541, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606064.67963875, + "first": 3606085.517932916, + "last": 3606095.333423208, + "chunks": 76 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 25.956132195884578, + "seconds": 9.824268041, + "prefill_seconds": 0.375099959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 31.028073875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606064.679766916, + "first": 3606095.708514583, + "last": 3606105.5327065, + "chunks": 76 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 25.998247972971406, + "seconds": 9.80835325, + "prefill_seconds": 0.377797958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.378097291, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606064.67970125, + "first": 3606065.058400625, + "last": 3606074.866651333, + "chunks": 76 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 25.735948028532654, + "seconds": 9.908319667, + "prefill_seconds": 0.370257542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 10.556647125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606064.679569333, + "first": 3606075.236987916, + "last": 3606085.145233291, + "chunks": 76 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 31.075094534881394, + "ttft_max": 0.20538291614502668, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.072944666747166, + "seconds": 8.206496125, + "prefill_seconds": 0.203727334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.204398334, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 365, + "accepted": 200, + "acceptance_rate": 0.547945205479452, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3606141.369936875, + "first": 3606141.575319791, + "last": 3606149.781248166, + "chunks": 56 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 41.92490293167692, + "ttft_max": 0.9278107918798923, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.826532270883073, + "seconds": 23.553248041, + "prefill_seconds": 0.228371666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228656208, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 66, + "drafted": 247, + "accepted": 189, + "acceptance_rate": 0.7651821862348178, + "tokens_per_round": 3.878787878787879 + }, + "sent": 3606149.7824475, + "first": 3606150.011670291, + "last": 3606173.564840375, + "chunks": 67 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.93183481567368, + "seconds": 23.326367833, + "prefill_seconds": 0.455304625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.455602625, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 66, + "drafted": 245, + "accepted": 189, + "acceptance_rate": 0.7714285714285715, + "tokens_per_round": 3.878787878787879 + }, + "sent": 3606149.782563416, + "first": 3606150.238595125, + "last": 3606173.564934041, + "chunks": 67 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.791077036134208, + "seconds": 23.63063475, + "prefill_seconds": 0.698629208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.927254084, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 69, + "drafted": 236, + "accepted": 186, + "acceptance_rate": 0.788135593220339, + "tokens_per_round": 3.710144927536232 + }, + "sent": 3606149.782507666, + "first": 3606150.710318458, + "last": 3606174.340885833, + "chunks": 70 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.683139854491918, + "seconds": 23.869387041, + "prefill_seconds": 0.68825425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.688510666, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 69, + "drafted": 239, + "accepted": 186, + "acceptance_rate": 0.7782426778242678, + "tokens_per_round": 3.710144927536232 + }, + "sent": 3606149.782372166, + "first": 3606150.471545041, + "last": 3606174.340864916, + "chunks": 70 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 21.680902825813735, + "ttft_max": 0.4451714172028005, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 21.680688840498675, + "seconds": 11.761618917, + "prefill_seconds": 0.444441917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.444756875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606174.342055833, + "first": 3606174.78722725, + "last": 3606186.548730083, + "chunks": 76 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 35.18775727391821, + "ttft_max": 1.921789416577667, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.267201928012476, + "seconds": 27.516396209, + "prefill_seconds": 1.4708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.92109, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 94, + "drafted": 224, + "accepted": 161, + "acceptance_rate": 0.71875, + "tokens_per_round": 2.723404255319149 + }, + "sent": 3606186.550534833, + "first": 3606188.47232425, + "last": 3606215.988622041, + "chunks": 95 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 8.900161211192339, + "seconds": 28.651166417, + "prefill_seconds": 0.450010667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.4503955, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 92, + "drafted": 237, + "accepted": 163, + "acceptance_rate": 0.6877637130801688, + "tokens_per_round": 2.782608695652174 + }, + "sent": 3606186.550593958, + "first": 3606187.001462875, + "last": 3606215.65254575, + "chunks": 93 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 8.942051678556043, + "seconds": 28.516945458, + "prefill_seconds": 0.47025075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.920595625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 94, + "drafted": 245, + "accepted": 161, + "acceptance_rate": 0.6571428571428571, + "tokens_per_round": 2.723404255319149 + }, + "sent": 3606186.550662875, + "first": 3606187.47172825, + "last": 3606215.988588166, + "chunks": 95 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.103197704543579, + "seconds": 28.012134667, + "prefill_seconds": 0.975065292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.425407291, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 94, + "drafted": 234, + "accepted": 161, + "acceptance_rate": 0.688034188034188, + "tokens_per_round": 2.723404255319149 + }, + "sent": 3606186.550445166, + "first": 3606187.976557666, + "last": 3606215.988817541, + "chunks": 95 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 32.82984563370725, + "ttft_max": 0.2014766251668334, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 32.83012336049014, + "seconds": 7.767256833, + "prefill_seconds": 0.200702875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200988792, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 354, + "accepted": 200, + "acceptance_rate": 0.5649717514124294, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3606225.58774575, + "first": 3606225.789222375, + "last": 3606233.556544916, + "chunks": 56 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 39.476843897617066, + "ttft_max": 0.9977558329701424, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.06532199933952, + "seconds": 25.334509916, + "prefill_seconds": 0.743525709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.743825583, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 64, + "drafted": 241, + "accepted": 191, + "acceptance_rate": 0.7925311203319502, + "tokens_per_round": 4.0 + }, + "sent": 3606233.558710833, + "first": 3606234.30318025, + "last": 3606259.637624083, + "chunks": 65 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.966010124797743, + "seconds": 25.586969791, + "prefill_seconds": 0.491034333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.491316208, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 64, + "drafted": 242, + "accepted": 191, + "acceptance_rate": 0.7892561983471075, + "tokens_per_round": 4.0 + }, + "sent": 3606233.558772708, + "first": 3606234.050711375, + "last": 3606259.63764625, + "chunks": 65 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.869207552850355, + "seconds": 25.83794075, + "prefill_seconds": 0.240055, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.240312917, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 64, + "drafted": 244, + "accepted": 191, + "acceptance_rate": 0.7827868852459017, + "tokens_per_round": 4.0 + }, + "sent": 3606233.558825416, + "first": 3606233.799724416, + "last": 3606259.637587541, + "chunks": 65 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.166927138522293, + "seconds": 25.081324625, + "prefill_seconds": 0.75667025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.996916125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 64, + "drafted": 239, + "accepted": 191, + "acceptance_rate": 0.799163179916318, + "tokens_per_round": 4.0 + }, + "sent": 3606233.558633875, + "first": 3606234.556389708, + "last": 3606259.637656208, + "chunks": 65 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 19.40846506022387, + "ttft_max": 0.4472993751987815, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 19.408321894553616, + "seconds": 13.138693875, + "prefill_seconds": 0.446586166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.4468845, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 74, + "drafted": 471, + "accepted": 181, + "acceptance_rate": 0.3842887473460722, + "tokens_per_round": 3.4594594594594597 + }, + "sent": 3606259.639965625, + "first": 3606260.087265, + "last": 3606273.225861958, + "chunks": 75 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 37.247923082251035, + "ttft_max": 2.0259827501140535, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.487299469359217, + "seconds": 26.878038458, + "prefill_seconds": 1.035085542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.035508875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 233, + "accepted": 159, + "acceptance_rate": 0.6824034334763949, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606273.227919, + "first": 3606274.264037375, + "last": 3606301.141950958, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.66677787410069, + "seconds": 26.379006875, + "prefill_seconds": 1.005173042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.534582791, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 229, + "accepted": 159, + "acceptance_rate": 0.6943231441048034, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606273.228045125, + "first": 3606274.76313025, + "last": 3606301.141967875, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.311984043601145, + "seconds": 27.384067542, + "prefill_seconds": 0.529000083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.529438167, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 233, + "accepted": 159, + "acceptance_rate": 0.6824034334763949, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606273.227982375, + "first": 3606273.757919333, + "last": 3606301.141920083, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.85000733950893, + "seconds": 25.888305583, + "prefill_seconds": 1.495880584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.02529125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 228, + "accepted": 159, + "acceptance_rate": 0.6973684210526315, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606273.227833291, + "first": 3606275.253816041, + "last": 3606301.1419965, + "chunks": 97 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 35.350466980056616, + "ttft_max": 0.20126425009220839, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 35.3500488025168, + "seconds": 7.213568542, + "prefill_seconds": 0.200582458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200864167, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 54, + "drafted": 355, + "accepted": 201, + "acceptance_rate": 0.5661971830985916, + "tokens_per_round": 4.7407407407407405 + }, + "sent": 3606311.548035666, + "first": 3606311.749299916, + "last": 3606318.962783125, + "chunks": 55 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 40.423617449409754, + "ttft_max": 0.8631084999069571, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.975599933959169, + "seconds": 23.233354125, + "prefill_seconds": 0.207475292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.20785825, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 243, + "accepted": 190, + "acceptance_rate": 0.7818930041152263, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3606318.964425125, + "first": 3606319.172912041, + "last": 3606342.406194541, + "chunks": 66 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.398315459161685, + "seconds": 24.523202917, + "prefill_seconds": 0.424516458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.424947459, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 68, + "drafted": 244, + "accepted": 187, + "acceptance_rate": 0.7663934426229508, + "tokens_per_round": 3.764705882352941 + }, + "sent": 3606318.964586583, + "first": 3606319.389932416, + "last": 3606343.913043416, + "chunks": 69 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.375067787026046, + "seconds": 24.578152667, + "prefill_seconds": 0.654687291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.862650709, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 234, + "accepted": 185, + "acceptance_rate": 0.7905982905982906, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606318.964503083, + "first": 3606319.827611583, + "last": 3606344.405685958, + "chunks": 71 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.283350014062258, + "seconds": 24.797366583, + "prefill_seconds": 0.435474292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.643465125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 236, + "accepted": 185, + "acceptance_rate": 0.7838983050847458, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606318.964343125, + "first": 3606319.608362333, + "last": 3606344.405664041, + "chunks": 71 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.181142415827715, + "ttft_max": 0.4115130412392318, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.18076293026933, + "seconds": 11.00050075, + "prefill_seconds": 0.410683709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.410999875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 468, + "accepted": 180, + "acceptance_rate": 0.38461538461538464, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606344.407623125, + "first": 3606344.819136166, + "last": 3606355.819456833, + "chunks": 76 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 41.04567326689068, + "ttft_max": 1.7338110408745706, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.261494365126726, + "seconds": 24.85018175, + "prefill_seconds": 0.422173833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.422573417, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 223, + "accepted": 158, + "acceptance_rate": 0.7085201793721974, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606355.821066083, + "first": 3606356.244260083, + "last": 3606381.094373708, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.446103423829815, + "seconds": 24.411016209, + "prefill_seconds": 0.439312583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.861861041, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 223, + "accepted": 158, + "acceptance_rate": 0.7085201793721974, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606355.821124958, + "first": 3606356.683676166, + "last": 3606381.094626, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.64010986129906, + "seconds": 23.965917958, + "prefill_seconds": 0.884405125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.30694925, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 219, + "accepted": 158, + "acceptance_rate": 0.7214611872146118, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606355.821194625, + "first": 3606357.128658333, + "last": 3606381.094574708, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.832718965943481, + "seconds": 23.5397965, + "prefill_seconds": 1.310551042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.7330955, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 219, + "accepted": 158, + "acceptance_rate": 0.7214611872146118, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606355.8209675, + "first": 3606357.554778541, + "last": 3606381.094475583, + "chunks": 98 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 28.223626529326367, + "ttft_max": 0.2059693750925362, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 28.223380789937625, + "seconds": 9.035062167, + "prefill_seconds": 0.205207625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.205514375, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 376, + "accepted": 202, + "acceptance_rate": 0.5372340425531915, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3606391.650607583, + "first": 3606391.856576958, + "last": 3606400.891560458, + "chunks": 54 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 28.761086081267262, + "ttft_max": 27.405207625124604, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 28.326025086672857, + "seconds": 9.002322042, + "prefill_seconds": 0.273832209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.274136208, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 376, + "accepted": 202, + "acceptance_rate": 0.5372340425531915, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3606400.894081041, + "first": 3606401.168842916, + "last": 3606410.171049916, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 29.82616056363124, + "seconds": 8.549541583, + "prefill_seconds": 0.265464208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 18.628984333, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 368, + "accepted": 202, + "acceptance_rate": 0.5489130434782609, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3606400.893924166, + "first": 3606419.523642625, + "last": 3606428.073088041, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 30.596510163274633, + "seconds": 8.334283833, + "prefill_seconds": 0.226095667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 27.404542583, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 368, + "accepted": 202, + "acceptance_rate": 0.5489130434782609, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3606400.894007083, + "first": 3606428.299214708, + "last": 3606436.633428541, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 28.817921085343936, + "seconds": 8.8486605, + "prefill_seconds": 0.238501375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 9.51489725, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 52, + "drafted": 377, + "accepted": 203, + "acceptance_rate": 0.5384615384615384, + "tokens_per_round": 4.923076923076923 + }, + "sent": 3606400.893823, + "first": 3606410.409605375, + "last": 3606419.258174583, + "chunks": 53 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 22.158009935284845, + "ttft_max": 0.43920179177075624, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 22.157811137460392, + "seconds": 11.508356959, + "prefill_seconds": 0.438467209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.438798917, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 459, + "accepted": 180, + "acceptance_rate": 0.39215686274509803, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606436.634665083, + "first": 3606437.073866875, + "last": 3606448.582120583, + "chunks": 76 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 36.93984744629104, + "ttft_max": 2.149744499940425, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.849580616640145, + "seconds": 25.889427167, + "prefill_seconds": 1.308605375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.149030417, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 227, + "accepted": 159, + "acceptance_rate": 0.7004405286343612, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606448.584418208, + "first": 3606450.734162708, + "last": 3606476.623452708, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.531338599247102, + "seconds": 26.753849666, + "prefill_seconds": 0.444140084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.284677042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 231, + "accepted": 159, + "acceptance_rate": 0.6883116883116883, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606448.584539041, + "first": 3606449.869655166, + "last": 3606476.623400333, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.688840220673356, + "seconds": 26.318939542, + "prefill_seconds": 0.879069125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.71950625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 227, + "accepted": 159, + "acceptance_rate": 0.7004405286343612, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606448.58448275, + "first": 3606450.304553666, + "last": 3606476.62342825, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.234971686120044, + "seconds": 27.612429, + "prefill_seconds": 0.425563125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.425944542, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 99, + "drafted": 227, + "accepted": 156, + "acceptance_rate": 0.6872246696035242, + "tokens_per_round": 2.585858585858586 + }, + "sent": 3606448.584335333, + "first": 3606449.010994333, + "last": 3606476.623366333, + "chunks": 100 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 34.74238473493653, + "ttft_max": 0.20315800001844764, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 34.741093727197175, + "seconds": 7.340010709, + "prefill_seconds": 0.202288333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.202571875, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 355, + "accepted": 200, + "acceptance_rate": 0.5633802816901409, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3606487.465374333, + "first": 3606487.668532333, + "last": 3606495.008270291, + "chunks": 56 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 45.67148543407574, + "ttft_max": 0.8624916248954833, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.417828712467486, + "seconds": 22.333493208, + "prefill_seconds": 0.199364125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.199731458, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 231, + "accepted": 186, + "acceptance_rate": 0.8051948051948052, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606495.010333583, + "first": 3606495.210814833, + "last": 3606517.544128833, + "chunks": 71 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.529024323617588, + "seconds": 22.118090208, + "prefill_seconds": 0.41482, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.415123, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 227, + "accepted": 186, + "acceptance_rate": 0.8193832599118943, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606495.010474125, + "first": 3606495.426240125, + "last": 3606517.5441555, + "chunks": 71 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.766654527744414, + "seconds": 21.671410459, + "prefill_seconds": 0.662173417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.861817625, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 70, + "drafted": 224, + "accepted": 186, + "acceptance_rate": 0.8303571428571429, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606495.010416916, + "first": 3606495.872908541, + "last": 3606517.544200166, + "chunks": 71 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.64494984033345, + "seconds": 21.897904542, + "prefill_seconds": 0.435676917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.6354045, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 225, + "accepted": 186, + "acceptance_rate": 0.8266666666666667, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606495.010198375, + "first": 3606495.646404666, + "last": 3606517.544224625, + "chunks": 71 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.125861831112022, + "ttft_max": 0.43125058291479945, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.12566704818981, + "seconds": 11.026708958, + "prefill_seconds": 0.430470709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.430793375, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 448, + "accepted": 180, + "acceptance_rate": 0.4017857142857143, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606517.5461745, + "first": 3606517.977425083, + "last": 3606529.004041166, + "chunks": 76 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 40.05300163872442, + "ttft_max": 1.7167558339424431, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.360270922105201, + "seconds": 24.613255958, + "prefill_seconds": 0.852961125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.289912375, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 218, + "accepted": 158, + "acceptance_rate": 0.7247706422018348, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606529.005617333, + "first": 3606530.296233, + "last": 3606554.909576625, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.542903217856782, + "seconds": 24.186886167, + "prefill_seconds": 1.279218041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.716137417, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 217, + "accepted": 158, + "acceptance_rate": 0.728110599078341, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606529.005735666, + "first": 3606530.7224915, + "last": 3606554.909267916, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.185256518927337, + "seconds": 25.036188291, + "prefill_seconds": 0.866474084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.866868041, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 222, + "accepted": 158, + "acceptance_rate": 0.7117117117117117, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606529.005681125, + "first": 3606529.873125625, + "last": 3606554.90949975, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.013385130654662, + "seconds": 25.465913542, + "prefill_seconds": 0.4365775, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.436960042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 222, + "accepted": 158, + "acceptance_rate": 0.7117117117117117, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606529.005529375, + "first": 3606529.443320458, + "last": 3606554.909149625, + "chunks": 98 + } + ] + } + ] + }, + "M5 Ultra": { + "functional": { + "resumed": { + "usage": { + "prompt_tokens": 3605, + "completion_tokens": 64, + "total_tokens": 3669, + "prompt_tokens_details": { + "cached_tokens": 3517 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 545.6868379633086, + "seconds": 0.115450833, + "prefill_seconds": 0.0341725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034537708, + "sampling": "greedy", + "drafts": true, + "token_sha": "2fed3f850c53", + "token_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91", + "min_rows": 2 + }, + "speculative": { + "rounds": 11, + "drafted": 58, + "accepted": 52, + "acceptance_rate": 0.896551724137931, + "tokens_per_round": 5.818181818181818 + }, + "wall": 0.15039120900000036 + }, + "resume_request": { + "messages": [ + { + "role": "user" + }, + { + "role": "assistant" + }, + { + "role": "user" + } + ], + "max_tokens": 64 + }, + "stress_reference_hashes": [ + { + "prompt": "quicksort-raw", + "sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0" + }, + { + "prompt": "rust-reverse-words", + "sha256": "d9499a1af105f9d0477f4fda4f2afde556c782008fe552d4a7cd0ebf71e80101" + }, + { + "prompt": "multiply-17-23", + "sha256": "72bba60a60f4df7bf0b2a34dab50868cd071c0b99c2ac9d3b2f0cc99a8c6ab84" + }, + { + "prompt": "sleep-tips", + "sha256": "dc9d14472ff7c82357f0686ee61236b54726924051b1222778247438e54345db" + }, + { + "prompt": "js-debounce-raw", + "sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458" + }, + { + "prompt": "linear-equation", + "sha256": "8ebb10c3e63f3fd3a265072c4a9ef09d88058a4fc50e488f523ce44cb0afdc0d" + }, + { + "prompt": "fishing-village", + "sha256": "c1100e452897c3c18c70f946d0ae4f05a345e7f90b10bece58e0cd584daf0ade" + }, + { + "prompt": "sql-left-join", + "sha256": "d04f354206e1c4c56ff5956161df55ffea050df8d858b655c26a2f8673b741b4" + } + ], + "stress": [ + { + "slots": 6, + "seconds": 60, + "completed": 453, + "cancelled": 247, + "wrong": 0, + "errors": [] + }, + { + "slots": 16, + "seconds": 90, + "completed": 852, + "cancelled": 459, + "wrong": 0, + "errors": [] + } + ], + "sampled": [ + { + "streams": 1, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ] + ] + }, + { + "streams": 4, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ] + ] + }, + { + "streams": 8, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ] + ] + }, + { + "streams": 16, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ], + [ + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986" + ], + [ + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351" + ], + [ + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369" + ], + [ + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a" + ], + [ + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363" + ], + [ + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398" + ], + [ + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db" + ], + [ + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd" + ] + ] + } + ], + "fresh": { + "usage": { + "prompt_tokens": 3605, + "completion_tokens": 64, + "total_tokens": 3669, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 597.1205482317299, + "seconds": 0.105506334, + "prefill_seconds": 0.308241792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.310870084, + "sampling": "greedy", + "drafts": true, + "token_sha": "2fed3f850c53", + "token_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91", + "min_rows": 2 + }, + "speculative": { + "rounds": 10, + "drafted": 59, + "accepted": 53, + "acceptance_rate": 0.8983050847457628, + "tokens_per_round": 6.4 + }, + "wall": 0.41713370900001223 + }, + "resume_equal": true + }, + "long-context": [ + { + "length": 4096, + "plain": { + "usage": { + "prompt_tokens": 4096, + "completion_tokens": 64, + "total_tokens": 4160, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 251.27072892278412, + "seconds": 0.250725583, + "prefill_seconds": 0.298777459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.299359667, + "sampling": "greedy", + "drafts": false, + "token_sha": "8394ececc0cc", + "token_sha256": "8394ececc0ccdeea4fd63598ff8309e489e7309932166b11e366f6706b8a4b3a", + "min_rows": 1 + }, + "speculative": { + "rounds": 63, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.0158730158730158 + }, + "wall": 0.5511507499999997 + }, + "drafted": { + "usage": { + "prompt_tokens": 4096, + "completion_tokens": 64, + "total_tokens": 4160, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 313.5117331542196, + "seconds": 0.200949417, + "prefill_seconds": 0.302226708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.302388917, + "sampling": "greedy", + "drafts": true, + "token_sha": "8394ececc0cc", + "token_sha256": "8394ececc0ccdeea4fd63598ff8309e489e7309932166b11e366f6706b8a4b3a", + "min_rows": 2 + }, + "speculative": { + "rounds": 26, + "drafted": 68, + "accepted": 38, + "acceptance_rate": 0.5588235294117647, + "tokens_per_round": 2.4615384615384617 + }, + "wall": 0.5038303750000002 + }, + "equal": true + }, + { + "length": 16384, + "plain": { + "usage": { + "prompt_tokens": 16384, + "completion_tokens": 64, + "total_tokens": 16448, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 231.32519503146526, + "seconds": 0.272343875, + "prefill_seconds": 1.283367667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.283625625, + "sampling": "greedy", + "drafts": false, + "token_sha": "a2dcb347e183", + "token_sha256": "a2dcb347e183d9e8a0be485abb99d222ff060dfc83781384863e7bf16cf22da3", + "min_rows": 1 + }, + "speculative": { + "rounds": 63, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.0158730158730158 + }, + "wall": 1.5566373330000003 + }, + "drafted": { + "usage": { + "prompt_tokens": 16384, + "completion_tokens": 64, + "total_tokens": 16448, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 259.7309798859134, + "seconds": 0.242558666, + "prefill_seconds": 1.342481375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.34274125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a2dcb347e183", + "token_sha256": "a2dcb347e183d9e8a0be485abb99d222ff060dfc83781384863e7bf16cf22da3", + "min_rows": 2 + }, + "speculative": { + "rounds": 32, + "drafted": 57, + "accepted": 31, + "acceptance_rate": 0.543859649122807, + "tokens_per_round": 2.0 + }, + "wall": 1.5860129589999996 + }, + "equal": true + }, + { + "length": 65536, + "plain": { + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 64, + "total_tokens": 65600, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.88251974236329, + "seconds": 0.399030875, + "prefill_seconds": 7.318780875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.319519292, + "sampling": "greedy", + "drafts": false, + "token_sha": "c2ba0e4604c6", + "token_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "min_rows": 1 + }, + "speculative": { + "rounds": 63, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.0158730158730158 + }, + "wall": 7.720276332999999 + }, + "drafted": { + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 64, + "total_tokens": 65600, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 151.86195081442588, + "seconds": 0.414850459, + "prefill_seconds": 7.556797125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.557524875, + "sampling": "greedy", + "drafts": true, + "token_sha": "c2ba0e4604c6", + "token_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "min_rows": 2 + }, + "speculative": { + "rounds": 36, + "drafted": 122, + "accepted": 27, + "acceptance_rate": 0.22131147540983606, + "tokens_per_round": 1.7777777777777777 + }, + "wall": 7.974062624999998 + }, + "equal": true + }, + { + "length": 130816, + "plain": { + "usage": { + "prompt_tokens": 130816, + "completion_tokens": 64, + "total_tokens": 130880, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 116.19828729405333, + "seconds": 0.542176666, + "prefill_seconds": 20.563776416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 20.565105666, + "sampling": "greedy", + "drafts": false, + "token_sha": "3cc988610216", + "token_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "min_rows": 1 + }, + "speculative": { + "rounds": 63, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.0158730158730158 + }, + "wall": 21.110205083000004 + }, + "drafted": { + "usage": { + "prompt_tokens": 130816, + "completion_tokens": 64, + "total_tokens": 130880, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 230.23142702247748, + "seconds": 0.273637708, + "prefill_seconds": 21.036062792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 21.037378792, + "sampling": "greedy", + "drafts": true, + "token_sha": "3cc988610216", + "token_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "min_rows": 5 + }, + "speculative": { + "rounds": 12, + "drafted": 108, + "accepted": 51, + "acceptance_rate": 0.4722222222222222, + "tokens_per_round": 5.333333333333333 + }, + "wall": 21.314060583999996 + }, + "equal": true + } + ], + "performance-lfm2": [ + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.9714374495252, + "ttft_max": 0.016745999999997707, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 289.9559952775242, + "seconds": 0.879443792, + "prefill_seconds": 0.016395833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016501792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 124.128642, + "first": 124.145388, + "last": 125.024784958, + "chunks": 217 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 652.2080388699854, + "ttft_max": 0.06611995800000159, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.96049804643943, + "seconds": 1.527307375, + "prefill_seconds": 0.01635325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049324708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 125.0255105, + "first": 125.075392583, + "last": 126.602660541, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 168.34147176327625, + "seconds": 1.51477825, + "prefill_seconds": 0.01636675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0656875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 125.025627875, + "first": 125.091747833, + "last": 126.606481416, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.45264548507336, + "seconds": 1.560084875, + "prefill_seconds": 0.016379542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016536916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 125.025571583, + "first": 125.04256325, + "last": 126.602603041, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.19216464411534, + "seconds": 1.54365675, + "prefill_seconds": 0.016455834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.032961709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 125.025436875, + "first": 125.059020333, + "last": 126.602634583, + "chunks": 224 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 830.9301245588248, + "ttft_max": 0.13169079099999692, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.19389804837425, + "seconds": 2.447360208, + "prefill_seconds": 0.016465708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016684167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607762625, + "first": 126.625097166, + "last": 129.0724045, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.89724081177302, + "seconds": 2.4309505, + "prefill_seconds": 0.016419292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033021917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607816791, + "first": 126.641500916, + "last": 129.072416125, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.5459214960778, + "seconds": 2.349236125, + "prefill_seconds": 0.016382917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114713417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.608144291, + "first": 126.7233725, + "last": 129.072579958, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.79394994721177, + "seconds": 2.365624417, + "prefill_seconds": 0.016353916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098507667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607865791, + "first": 126.70698675, + "last": 129.072642708, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.05374686342388, + "seconds": 2.381981084, + "prefill_seconds": 0.016367334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082089875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.608013291, + "first": 126.690629583, + "last": 129.072869166, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.60819370489229, + "seconds": 2.414585375, + "prefill_seconds": 0.0163805, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049418458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607912666, + "first": 126.657892958, + "last": 129.072431375, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.95296996983765, + "seconds": 2.340459375, + "prefill_seconds": 0.016380709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131116792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 126.60806725, + "first": 126.739758041, + "last": 129.08017725, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.3282934211501, + "seconds": 2.398232792, + "prefill_seconds": 0.016363417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065817125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607685583, + "first": 126.674259291, + "last": 129.072453583, + "chunks": 235 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 828.3957621161079, + "ttft_max": 0.26417679200000066, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.4837114578651, + "seconds": 4.858650292, + "prefill_seconds": 0.016387708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082543416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082258416, + "first": 129.166236416, + "last": 134.025300916, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.132941351539856, + "seconds": 4.891341125, + "prefill_seconds": 0.016411458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049863042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082833125, + "first": 129.133453208, + "last": 134.024820291, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.0191678536274, + "seconds": 4.809581333, + "prefill_seconds": 0.016347333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131707875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082449458, + "first": 129.215360125, + "last": 134.025603666, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.66131800556297, + "seconds": 4.842263917, + "prefill_seconds": 0.016385584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098904375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082311083, + "first": 129.182623833, + "last": 134.025126708, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.78590461813402, + "seconds": 4.924119833, + "prefill_seconds": 0.016984208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0171665, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082358166, + "first": 129.100618625, + "last": 134.024723791, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.83961379629883, + "seconds": 4.82592475, + "prefill_seconds": 0.016392042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.115322833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.0824055, + "first": 129.199020583, + "last": 134.025367041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.93609398148886, + "seconds": 4.727817333, + "prefill_seconds": 0.016383958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212811334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082654708, + "first": 129.297255958, + "last": 134.025415541, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.56612729160864, + "seconds": 4.760471083, + "prefill_seconds": 0.016407792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.180273792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082776333, + "first": 129.264506041, + "last": 134.025465958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.30800705704185, + "seconds": 4.874970666, + "prefill_seconds": 0.016384042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0662055, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082710958, + "first": 129.1498505, + "last": 134.0255115, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.959096956934374, + "seconds": 4.907706541, + "prefill_seconds": 0.016405084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033492417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082581166, + "first": 129.117039416, + "last": 134.024706166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.38055057810182, + "seconds": 4.777020792, + "prefill_seconds": 0.016359792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.164066458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082493833, + "first": 129.248097666, + "last": 134.025556375, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.750243175321984, + "seconds": 4.744164583, + "prefill_seconds": 0.016359167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196493458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.083095208, + "first": 129.2808835, + "last": 134.025245041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.12412620540813, + "seconds": 4.711392458, + "prefill_seconds": 0.0163735, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.229135875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082913458, + "first": 129.313639791, + "last": 134.025751041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.19990749800366, + "seconds": 4.793241417, + "prefill_seconds": 0.016356667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.147830375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082537041, + "first": 129.231729, + "last": 134.025192166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.31266282813846, + "seconds": 4.695037708, + "prefill_seconds": 0.016368, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.245525542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.083007458, + "first": 129.330005458, + "last": 134.025707458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.502041390310886, + "seconds": 4.678723833, + "prefill_seconds": 0.016355791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.261876959, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082185333, + "first": 129.346362125, + "last": 134.02580075, + "chunks": 240 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 261.8548695057814, + "ttft_max": 0.019914957999986882, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 261.84384531549165, + "seconds": 0.973862875, + "prefill_seconds": 0.019466083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019607666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 134.028428208, + "first": 134.048343166, + "last": 135.022165041, + "chunks": 248 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 623.8913057741527, + "ttft_max": 0.07612033399999518, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.97151868320893, + "seconds": 1.634913875, + "prefill_seconds": 0.018871917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019013209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 135.024678125, + "first": 135.044165708, + "last": 136.679035208, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.655644687928, + "seconds": 1.5971875, + "prefill_seconds": 0.018885958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.056715125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 135.024729791, + "first": 135.081912541, + "last": 136.679057833, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.79077236147523, + "seconds": 1.616064084, + "prefill_seconds": 0.018858833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.037831417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 135.024784708, + "first": 135.06302475, + "last": 136.679047833, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.55734099774537, + "seconds": 1.578386958, + "prefill_seconds": 0.0188095, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075517208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 135.024608416, + "first": 135.10072875, + "last": 136.679065875, + "chunks": 249 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 804.7903304448168, + "ttft_max": 0.15244674999999575, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.45497930497726, + "seconds": 2.4180935, + "prefill_seconds": 0.018895542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132696, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682299791, + "first": 136.816032333, + "last": 139.234285625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.50799246585228, + "seconds": 2.512117458, + "prefill_seconds": 0.018809333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038682083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682349875, + "first": 136.721842625, + "last": 139.233927375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.2725854447154, + "seconds": 2.493336791, + "prefill_seconds": 0.018816833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057453708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682500916, + "first": 136.740661625, + "last": 139.233946833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.75387442870228, + "seconds": 2.530920041, + "prefill_seconds": 0.019629541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019855542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682447458, + "first": 136.70303025, + "last": 139.23390925, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.1153164506986, + "seconds": 2.403046125, + "prefill_seconds": 0.018809541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151517333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 136.682397375, + "first": 136.834844125, + "last": 139.237851958, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.04935504488036, + "seconds": 2.474542416, + "prefill_seconds": 0.018781333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076166791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682617583, + "first": 136.759446666, + "last": 139.234074125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.6375463110905, + "seconds": 2.43698375, + "prefill_seconds": 0.018829917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113811833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682559875, + "first": 136.797134958, + "last": 139.234247458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.83638983320756, + "seconds": 2.455786458, + "prefill_seconds": 0.018852292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094927917, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682232541, + "first": 136.778290083, + "last": 139.234064083, + "chunks": 252 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 798.7138944430546, + "ttft_max": 0.30219858300000624, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.038958238356024, + "seconds": 4.900174958, + "prefill_seconds": 0.018812083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225735791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.241876291, + "first": 139.469755916, + "last": 144.370635541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.645081133940984, + "seconds": 4.843757375, + "prefill_seconds": 0.018843625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.282080375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.241926416, + "first": 139.526226541, + "last": 144.370152958, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.114745571121645, + "seconds": 5.08832275, + "prefill_seconds": 0.018857834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038370542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242242916, + "first": 139.281345041, + "last": 144.3696365, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.250450280175464, + "seconds": 4.975566041, + "prefill_seconds": 0.018787792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150630333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242420083, + "first": 139.394317125, + "last": 144.370054333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.44518779537533, + "seconds": 4.956731833, + "prefill_seconds": 0.01885675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169326625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.241972833, + "first": 139.413183916, + "last": 144.370551041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.67643014217816, + "seconds": 5.031925084, + "prefill_seconds": 0.018838334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.09432575, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.2420175, + "first": 139.337810041, + "last": 144.37069625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.48755177359346, + "seconds": 5.050749958, + "prefill_seconds": 0.018804167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07552175, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242474, + "first": 139.318966, + "last": 144.369743, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.92971047107042, + "seconds": 5.107179625, + "prefill_seconds": 0.019435333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019633792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242060333, + "first": 139.262484125, + "last": 144.369621125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.64220612545368, + "seconds": 4.937821583, + "prefill_seconds": 0.018907209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188229709, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242191541, + "first": 139.432086583, + "last": 144.369915875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.058301350286555, + "seconds": 4.994290708, + "prefill_seconds": 0.018909833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.13187375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242326416, + "first": 139.375525041, + "last": 144.370594041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.840043719916856, + "seconds": 4.918977333, + "prefill_seconds": 0.0188405, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.207001916, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.24274875, + "first": 139.450944, + "last": 144.370281958, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.300370060335396, + "seconds": 5.069545208, + "prefill_seconds": 0.018811042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.056869542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.24210275, + "first": 139.300158791, + "last": 144.369966, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.8499599998342, + "seconds": 4.82498, + "prefill_seconds": 0.018821833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300614791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242850333, + "first": 139.545048916, + "last": 144.370439416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.44091554383292, + "seconds": 4.862615333, + "prefill_seconds": 0.018820458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.263346125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.2421485, + "first": 139.507378208, + "last": 144.370390166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.23848569029113, + "seconds": 4.8814585, + "prefill_seconds": 0.018789917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.24454025, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242551333, + "first": 139.488551541, + "last": 144.3701935, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.86513617793781, + "seconds": 5.013257, + "prefill_seconds": 0.018808041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113095333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.241802291, + "first": 139.35662025, + "last": 144.369869333, + "chunks": 252 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.72587991488484, + "ttft_max": 0.017593750000003183, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 289.6930751741181, + "seconds": 0.880241959, + "prefill_seconds": 0.016860375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017127334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 147.223546083, + "first": 147.241139833, + "last": 148.121282125, + "chunks": 217 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 649.6021903057296, + "ttft_max": 0.06667979200000218, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 162.79983373680727, + "seconds": 1.566340666, + "prefill_seconds": 0.017153125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017348375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 148.12660075, + "first": 148.144440958, + "last": 149.710751083, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 167.6232283395662, + "seconds": 1.521268875, + "prefill_seconds": 0.016347458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066252792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 148.126719708, + "first": 148.1933995, + "last": 149.714632708, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 164.51003069848784, + "seconds": 1.550057458, + "prefill_seconds": 0.0162855, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0336135, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 148.126664666, + "first": 148.16073, + "last": 149.710764916, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.2580215102581, + "seconds": 1.533760583, + "prefill_seconds": 0.0163095, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049878458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 148.126504958, + "first": 148.177041083, + "last": 149.710775, + "chunks": 224 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 829.8565857754016, + "ttft_max": 0.13205166699998472, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.7570662804786, + "seconds": 2.434203334, + "prefill_seconds": 0.016336958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033825708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719728875, + "first": 149.754335333, + "last": 152.188516791, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.46240456557089, + "seconds": 2.41792325, + "prefill_seconds": 0.016293041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.050100792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719781708, + "first": 149.770638708, + "last": 152.188533875, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.05825140106275, + "seconds": 2.4505505, + "prefill_seconds": 0.017265125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017462875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719938416, + "first": 149.737996958, + "last": 152.188544291, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.17673990755674, + "seconds": 2.401655958, + "prefill_seconds": 0.016282125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066412625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719830541, + "first": 149.786910916, + "last": 152.188621416, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.78403340480372, + "seconds": 2.344094, + "prefill_seconds": 0.016315416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131539833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 149.720132166, + "first": 149.852183833, + "last": 152.196253041, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.38048753710204, + "seconds": 2.352822042, + "prefill_seconds": 0.016322125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.115258875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.720000333, + "first": 149.835876291, + "last": 152.189067625, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.90075464692875, + "seconds": 2.385390083, + "prefill_seconds": 0.016310875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082632, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719877166, + "first": 149.8032345, + "last": 152.18876775, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.63435531635936, + "seconds": 2.369132042, + "prefill_seconds": 0.016312375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098949667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719657166, + "first": 149.819553125, + "last": 152.188947875, + "chunks": 235 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 828.6755735521535, + "ttft_max": 0.26237383299999806, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.976483494829445, + "seconds": 4.906064875, + "prefill_seconds": 0.016308208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033549875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.20229475, + "first": 152.237289541, + "last": 157.143305666, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.506561542949406, + "seconds": 4.678335833, + "prefill_seconds": 0.016273417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.261174, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202958583, + "first": 152.465332416, + "last": 157.143935, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.80460451006316, + "seconds": 4.922342375, + "prefill_seconds": 0.017116209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017279959, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202357791, + "first": 152.220975375, + "last": 157.143273958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.57511701564012, + "seconds": 4.759672292, + "prefill_seconds": 0.016269667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.179849709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202687208, + "first": 152.38391825, + "last": 157.144389083, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.31631183691749, + "seconds": 4.694722292, + "prefill_seconds": 0.016261083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244935209, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202408916, + "first": 152.44905775, + "last": 157.144159333, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.497494527736826, + "seconds": 4.857374667, + "prefill_seconds": 0.016310375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082430375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202455875, + "first": 152.286180208, + "last": 157.144008375, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.323544098349856, + "seconds": 4.873523084, + "prefill_seconds": 0.016281834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066054958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202730708, + "first": 152.269866833, + "last": 157.144050166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.392395322641754, + "seconds": 4.775961042, + "prefill_seconds": 0.016313, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163628, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202636541, + "first": 152.367643291, + "last": 157.1442535, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.75897038755569, + "seconds": 4.743394417, + "prefill_seconds": 0.016290917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196119792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202501708, + "first": 152.400212541, + "last": 157.144343833, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.2098882511113, + "seconds": 4.792342333, + "prefill_seconds": 0.01627575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.147333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202592625, + "first": 152.351327666, + "last": 157.144213208, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.85089150476571, + "seconds": 4.824894959, + "prefill_seconds": 0.016256834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114773875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202779791, + "first": 152.318726541, + "last": 157.143661333, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.129992851779946, + "seconds": 4.710881834, + "prefill_seconds": 0.016289166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228667375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202546791, + "first": 152.432780375, + "last": 157.144299583, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.0301257905778, + "seconds": 4.8085875, + "prefill_seconds": 0.016313, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131061416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202824, + "first": 152.335047458, + "last": 157.144494458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.149293060798676, + "seconds": 4.889807417, + "prefill_seconds": 0.016292833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.04980925, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202868833, + "first": 152.253582125, + "last": 157.143424291, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.67394686960922, + "seconds": 4.841102958, + "prefill_seconds": 0.016283458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098661333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.20291025, + "first": 152.302458083, + "last": 157.143725, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.94291697444283, + "seconds": 4.727219333, + "prefill_seconds": 0.016285708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212410958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202220541, + "first": 152.416500125, + "last": 157.143870666, + "chunks": 240 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 261.4389358917785, + "ttft_max": 0.01995562499999437, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 261.42741066011195, + "seconds": 0.975414167, + "prefill_seconds": 0.019505167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019642667, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 157.151165291, + "first": 157.171120916, + "last": 158.146492083, + "chunks": 248 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 621.9575399660765, + "ttft_max": 0.07646687499999416, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.14182827765322, + "seconds": 1.602344291, + "prefill_seconds": 0.018828959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057355417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 158.15290625, + "first": 158.210693208, + "last": 159.812987583, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.48832633280173, + "seconds": 1.6399945, + "prefill_seconds": 0.019593875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0197315, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 158.152959, + "first": 158.173025416, + "last": 159.812993458, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.03016363348462, + "seconds": 1.58355425, + "prefill_seconds": 0.018808125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076165958, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 158.153034708, + "first": 158.229501583, + "last": 159.81300875, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.29485342441458, + "seconds": 1.621159208, + "prefill_seconds": 0.018828042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038547125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 158.152837458, + "first": 158.191863208, + "last": 159.812975875, + "chunks": 249 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 803.6481953347434, + "ttft_max": 0.15192016599999647, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.61238418108967, + "seconds": 2.53447925, + "prefill_seconds": 0.019770458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01991025, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820251541, + "first": 159.840839125, + "last": 162.375272125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.36327475042472, + "seconds": 2.515704042, + "prefill_seconds": 0.018790042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038690417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820299, + "first": 159.859620833, + "last": 162.375289666, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.12461416990364, + "seconds": 2.496949458, + "prefill_seconds": 0.018780041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057416125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820439166, + "first": 159.878424416, + "last": 162.375407166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.47681621037697, + "seconds": 2.440732875, + "prefill_seconds": 0.018787542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113625458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820388875, + "first": 159.93475425, + "last": 162.3757295, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.93935860945503, + "seconds": 2.407037416, + "prefill_seconds": 0.018755042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151151375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 159.8203455, + "first": 159.972265666, + "last": 162.379263291, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.28566012405747, + "seconds": 2.42198225, + "prefill_seconds": 0.018747875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132366375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.82058875, + "first": 159.953500375, + "last": 162.375637625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.68018231006161, + "seconds": 2.459486416, + "prefill_seconds": 0.018772292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094867541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820525666, + "first": 159.915963208, + "last": 162.375684708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.89784024999332, + "seconds": 2.478186125, + "prefill_seconds": 0.018772834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076230041, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.82018225, + "first": 159.897197333, + "last": 162.37579275, + "chunks": 252 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 798.2060644244926, + "ttft_max": 0.30242166599998654, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.082243715282694, + "seconds": 5.091624917, + "prefill_seconds": 0.018790083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038452791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387256333, + "first": 162.426992083, + "last": 167.518563958, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.99798519348889, + "seconds": 4.904036167, + "prefill_seconds": 0.018809625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225475875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387305541, + "first": 162.614833375, + "last": 167.519646833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.21441254901633, + "seconds": 4.979067167, + "prefill_seconds": 0.018818959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150812333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387613125, + "first": 162.539740208, + "last": 167.51956775, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.39927234860693, + "seconds": 4.866479792, + "prefill_seconds": 0.018790792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.26276875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387708291, + "first": 162.652404833, + "last": 167.519421208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.64125397401233, + "seconds": 5.035420334, + "prefill_seconds": 0.018788459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094459, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387351458, + "first": 162.483348291, + "last": 167.519237833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.800133781219394, + "seconds": 4.922767209, + "prefill_seconds": 0.018728084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2068905, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387667333, + "first": 162.59601925, + "last": 167.519464416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.603122245333005, + "seconds": 4.941561458, + "prefill_seconds": 0.0187735, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188214166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387397375, + "first": 162.577278791, + "last": 167.519096166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.02185605900568, + "seconds": 4.997858167, + "prefill_seconds": 0.018797625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131972417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387799041, + "first": 162.520916083, + "last": 167.519039375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.453946602893055, + "seconds": 5.054114042, + "prefill_seconds": 0.018778125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075802416, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387568333, + "first": 162.46453425, + "last": 167.519609166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.197727318076026, + "seconds": 4.885270166, + "prefill_seconds": 0.018782167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2441205, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387998958, + "first": 162.633622416, + "last": 167.519282833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.80526185126259, + "seconds": 4.829064208, + "prefill_seconds": 0.018736541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300208875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387441125, + "first": 162.689862791, + "last": 167.51918725, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.40736524587146, + "seconds": 4.960378708, + "prefill_seconds": 0.018771959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169516292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387843791, + "first": 162.55851125, + "last": 167.519140791, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.26736818885785, + "seconds": 5.0728735, + "prefill_seconds": 0.018771375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057141208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387483916, + "first": 162.445761958, + "last": 167.518660041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.830627266884015, + "seconds": 5.0166605, + "prefill_seconds": 0.018771666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113197791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.3877535, + "first": 162.502124708, + "last": 167.519378458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.60158047773111, + "seconds": 4.847763084, + "prefill_seconds": 0.01871975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281475459, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387527541, + "first": 162.671121, + "last": 167.519336791, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.89828698936166, + "seconds": 5.110395875, + "prefill_seconds": 0.01938875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019613167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387183083, + "first": 162.408184791, + "last": 167.518552875, + "chunks": 252 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 345.7276309402865, + "ttft_max": 0.019197541000011142, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 345.65026843416723, + "seconds": 0.737739916, + "prefill_seconds": 0.01819825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.018542042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 2 + }, + "speculative": { + "rounds": 89, + "drafted": 306, + "accepted": 166, + "acceptance_rate": 0.5424836601307189, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 170.434448875, + "first": 170.453646416, + "last": 171.19122125, + "chunks": 90 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 633.5514893351391, + "ttft_max": 0.0739344999999787, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.98510734078445, + "seconds": 1.574218792, + "prefill_seconds": 0.0179905, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.054373459, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 171.217551833, + "first": 171.273054125, + "last": 172.84715575, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.54720369974092, + "seconds": 1.588317916, + "prefill_seconds": 0.017848833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036408458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + }, + "sent": 171.217838833, + "first": 171.255051541, + "last": 172.843258958, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.761572570732, + "seconds": 1.606182125, + "prefill_seconds": 0.018322083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.018604875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 171.217703708, + "first": 171.237184, + "last": 172.843268166, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 164.94290112406395, + "seconds": 1.545989541, + "prefill_seconds": 0.018218917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.072570042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 222, + "drafted": 41, + "accepted": 33, + "acceptance_rate": 0.8048780487804879, + "tokens_per_round": 1.1531531531531531 + }, + "sent": 171.217352083, + "first": 171.291286583, + "last": 172.83718675, + "chunks": 223 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 812.1249994900322, + "ttft_max": 0.13950241699998855, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.18747313821937, + "seconds": 2.4954135, + "prefill_seconds": 0.017388125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017560833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856445875, + "first": 172.874706416, + "last": 175.370184833, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.15704404096843, + "seconds": 2.424944542, + "prefill_seconds": 0.017013917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087911917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856716333, + "first": 172.945285125, + "last": 175.370540791, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.70128001877501, + "seconds": 2.458986041, + "prefill_seconds": 0.017051292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.054009708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856507541, + "first": 172.91115875, + "last": 175.370447958, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.67751940439814, + "seconds": 2.390381792, + "prefill_seconds": 0.01699375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138761167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 24, + "accepted": 18, + "acceptance_rate": 0.75, + "tokens_per_round": 1.080168776371308 + }, + "sent": 172.856785416, + "first": 172.996287833, + "last": 175.386635, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.98779025304378, + "seconds": 2.476021666, + "prefill_seconds": 0.019396792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036931875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856606041, + "first": 172.89410975, + "last": 175.370159291, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.43588215244323, + "seconds": 2.395808583, + "prefill_seconds": 0.017016833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.121752083, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 28, + "accepted": 21, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 172.856832125, + "first": 172.979291833, + "last": 175.375062791, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.42807278452388, + "seconds": 2.441872125, + "prefill_seconds": 0.017100292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.071068333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856558916, + "first": 172.928262041, + "last": 175.370236125, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.89930597631496, + "seconds": 2.407947792, + "prefill_seconds": 0.016990542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.104894583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856355, + "first": 172.962278375, + "last": 175.370590625, + "chunks": 234 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 794.6944987638544, + "ttft_max": 0.27339062500001887, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.011193681879966, + "seconds": 5.0988585, + "prefill_seconds": 0.017067916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.051142125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396549583, + "first": 175.449037666, + "last": 180.54798075, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.34534065594796, + "seconds": 5.065016875, + "prefill_seconds": 0.016973375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.085040708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396940291, + "first": 175.483047708, + "last": 180.548542, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.096281589595456, + "seconds": 4.894783125, + "prefill_seconds": 0.017031459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.254624417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396599125, + "first": 175.653388375, + "last": 180.548447583, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.91615750589105, + "seconds": 4.911765667, + "prefill_seconds": 0.017022167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2376825, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397019541, + "first": 175.636356583, + "last": 180.54875, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.55892191507427, + "seconds": 4.94579775, + "prefill_seconds": 0.017085083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.203777167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396647041, + "first": 175.602322416, + "last": 180.548898041, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.859778745725706, + "seconds": 5.013785083, + "prefill_seconds": 0.017029166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.136140291, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396692166, + "first": 175.534216125, + "last": 180.548619, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.51630928871303, + "seconds": 5.047874708, + "prefill_seconds": 0.017060833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.102066541, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397157708, + "first": 175.500112791, + "last": 180.548385375, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.38126740894436, + "seconds": 4.962898209, + "prefill_seconds": 0.0169445, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.186799916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397239166, + "first": 175.585225166, + "last": 180.548285625, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.27741474109795, + "seconds": 4.877823459, + "prefill_seconds": 0.016977833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.271606333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396979666, + "first": 175.670370291, + "last": 180.548828166, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.03211749181326, + "seconds": 4.996853208, + "prefill_seconds": 0.016988583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.152903666, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396747, + "first": 175.551213166, + "last": 180.548215, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.844554216135364, + "seconds": 5.115904917, + "prefill_seconds": 0.01705125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034082291, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396788666, + "first": 175.431957333, + "last": 180.547865708, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.68731040786727, + "seconds": 5.030844958, + "prefill_seconds": 0.017073458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119139792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396873458, + "first": 175.517188, + "last": 180.548184416, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.67913488754507, + "seconds": 5.132939625, + "prefill_seconds": 0.017062709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017221458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397074583, + "first": 175.41490575, + "last": 180.547812625, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.20732437437765, + "seconds": 4.979756375, + "prefill_seconds": 0.017061458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169880459, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396829125, + "first": 175.568279208, + "last": 180.548662125, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.73702453243566, + "seconds": 4.928772041, + "prefill_seconds": 0.017009958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.220804, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397466333, + "first": 175.619330208, + "last": 180.548715958, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.17800811704017, + "seconds": 5.081907584, + "prefill_seconds": 0.017031125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.068205584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396474958, + "first": 175.466075916, + "last": 180.548954125, + "chunks": 253 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 356.283045357237, + "ttft_max": 0.0198536249999961, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 356.2623676742896, + "seconds": 0.715764625, + "prefill_seconds": 0.019405291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019556458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 2 + }, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + }, + "sent": 180.559402541, + "first": 180.579256166, + "last": 181.29497925, + "chunks": 92 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 617.4407646534572, + "ttft_max": 0.07852779099999907, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 154.376093024738, + "seconds": 1.651810167, + "prefill_seconds": 0.019667583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019799375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 181.305110583, + "first": 181.325383083, + "last": 182.977196416, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.08517319160634, + "seconds": 1.6130545, + "prefill_seconds": 0.019354208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.058541542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 181.305161208, + "first": 181.364150666, + "last": 182.977168083, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 156.21058841312555, + "seconds": 1.63241175, + "prefill_seconds": 0.019416625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039194125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 181.305215083, + "first": 181.344788708, + "last": 182.977363416, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.00901903699273, + "seconds": 1.593660167, + "prefill_seconds": 0.019406792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0779435, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 181.305036, + "first": 181.383563791, + "last": 182.977279, + "chunks": 250 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 793.7909080606516, + "ttft_max": 0.15909400000001028, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 99.37856571901393, + "seconds": 2.565945666, + "prefill_seconds": 0.019888792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.020052083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988145125, + "first": 183.008811875, + "last": 185.574721083, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.20306644084567, + "seconds": 2.447144875, + "prefill_seconds": 0.019436125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138857125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988194666, + "first": 183.127767625, + "last": 185.575160208, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.99723881470433, + "seconds": 2.5248215, + "prefill_seconds": 0.019431667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.061271291, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988337291, + "first": 183.050054916, + "last": 185.5750405, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.5767566060795, + "seconds": 2.485943292, + "prefill_seconds": 0.019458459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.100055583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988463333, + "first": 183.0889355, + "last": 185.575254833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.7821996501377, + "seconds": 2.505349667, + "prefill_seconds": 0.019409708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.080589292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988240625, + "first": 183.069476125, + "last": 185.574807333, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.2295879774243, + "seconds": 2.544158917, + "prefill_seconds": 0.02180525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.041865, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988285916, + "first": 183.030620833, + "last": 185.574762083, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.38082320911384, + "seconds": 2.466608333, + "prefill_seconds": 0.019389542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119433209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988398041, + "first": 183.108331958, + "last": 185.575021041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.86764061650318, + "seconds": 2.431636666, + "prefill_seconds": 0.019388958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158255167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 6, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 182.988072916, + "first": 183.147166916, + "last": 185.578758166, + "chunks": 256 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 789.3346380570592, + "ttft_max": 0.3117172080000046, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.52943356426058, + "seconds": 5.148453791, + "prefill_seconds": 0.019454542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039253959, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590695791, + "first": 185.631209958, + "last": 190.779631666, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.870230469382534, + "seconds": 5.012754958, + "prefill_seconds": 0.019362375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.17457875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590743708, + "first": 185.767068708, + "last": 190.779988041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.904543704315714, + "seconds": 5.109755166, + "prefill_seconds": 0.019406417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077998375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591052625, + "first": 185.670025708, + "last": 190.780126208, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.479579513021015, + "seconds": 5.051547625, + "prefill_seconds": 0.01936975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.135929542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590789041, + "first": 185.728268083, + "last": 190.780533541, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.09559499773575, + "seconds": 5.090267917, + "prefill_seconds": 0.01942475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.097430375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.5912585, + "first": 185.689452666, + "last": 190.780443958, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.71594163114495, + "seconds": 5.1291395, + "prefill_seconds": 0.019402875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05859625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591108541, + "first": 185.65061025, + "last": 190.77987325, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.285914114393485, + "seconds": 4.877030541, + "prefill_seconds": 0.019357291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.309896666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591211708, + "first": 185.902928916, + "last": 190.780663958, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.07942152629365, + "seconds": 4.89636775, + "prefill_seconds": 0.019388875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.290604208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590877541, + "first": 185.88357275, + "last": 190.7804955, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.67053368265316, + "seconds": 4.9351145, + "prefill_seconds": 0.019424083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.251939834, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591358833, + "first": 185.844793958, + "last": 190.78015475, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.34316665682364, + "seconds": 5.167888834, + "prefill_seconds": 0.01960475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019844209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590833791, + "first": 185.611753583, + "last": 190.779611083, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.67403630212287, + "seconds": 5.032162792, + "prefill_seconds": 0.019429083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.155263208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591302958, + "first": 185.747701375, + "last": 190.780263916, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.26752849289349, + "seconds": 4.973908583, + "prefill_seconds": 0.019454792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.213334541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591161458, + "first": 185.80593725, + "last": 190.780202541, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.28697180415821, + "seconds": 5.070895917, + "prefill_seconds": 0.019438792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.116738916, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590919041, + "first": 185.708885791, + "last": 190.780083375, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.87409404691879, + "seconds": 4.9157485, + "prefill_seconds": 0.019384709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.271264208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591003125, + "first": 185.864176083, + "last": 190.780342958, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.06723171619991, + "seconds": 4.993417333, + "prefill_seconds": 0.019408791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.193930083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590959916, + "first": 185.786477416, + "last": 190.7806215, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.46810585048602, + "seconds": 4.954524667, + "prefill_seconds": 0.019427209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.232649, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.5906235, + "first": 185.825365833, + "last": 190.780392083, + "chunks": 255 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.5446835251194, + "ttft_max": 0.01664883299997655, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 289.52884856931587, + "seconds": 0.88074125, + "prefill_seconds": 0.016308417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016413334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 192.945268875, + "first": 192.961917708, + "last": 193.842610791, + "chunks": 217 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 649.3601917375305, + "ttft_max": 0.06672287500001062, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 162.73396034928217, + "seconds": 1.566974708, + "prefill_seconds": 0.017142042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017301542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 193.855370208, + "first": 193.873108583, + "last": 195.440038791, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 164.45010581596875, + "seconds": 1.550622292, + "prefill_seconds": 0.016355, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033608292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 193.85548125, + "first": 193.8894655, + "last": 195.440049791, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 167.56919930261643, + "seconds": 1.521759375, + "prefill_seconds": 0.016328708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066281583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 193.855425916, + "first": 193.922148791, + "last": 195.4438855, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.20392994080697, + "seconds": 1.53425975, + "prefill_seconds": 0.01636925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.04996875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 193.855305916, + "first": 193.905832083, + "last": 195.440065833, + "chunks": 224 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 830.0258741687314, + "ttft_max": 0.1323993340000129, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.77697039105169, + "seconds": 2.433740917, + "prefill_seconds": 0.016276833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033837958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456540916, + "first": 195.490997, + "last": 197.9247045, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.08126446027792, + "seconds": 2.450008667, + "prefill_seconds": 0.017435542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017601792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456586416, + "first": 195.474721708, + "last": 197.924690666, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.2004508521493, + "seconds": 2.40111975, + "prefill_seconds": 0.016355792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066483375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456443291, + "first": 195.523651708, + "last": 197.924731416, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.80566369748101, + "seconds": 2.343628, + "prefill_seconds": 0.016292792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131531875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 195.456493416, + "first": 195.58889275, + "last": 197.932476416, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.40519500956965, + "seconds": 2.352285792, + "prefill_seconds": 0.016349209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.115261166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456750041, + "first": 195.572604125, + "last": 197.92507575, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.92444446704627, + "seconds": 2.384861584, + "prefill_seconds": 0.016282792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082664833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456802458, + "first": 195.53992875, + "last": 197.925125416, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.48317214793134, + "seconds": 2.417447208, + "prefill_seconds": 0.016295375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.050117875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456674125, + "first": 195.50729325, + "last": 197.924719583, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.65957033632907, + "seconds": 2.368577166, + "prefill_seconds": 0.016295583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098914542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456369541, + "first": 195.55624475, + "last": 197.924863041, + "chunks": 235 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 831.5239140286818, + "ttft_max": 0.26237274999999727, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.768413276464976, + "seconds": 4.742561375, + "prefill_seconds": 0.016265, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.179626458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.945928541, + "first": 198.127401375, + "last": 202.870412958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.04038814956047, + "seconds": 4.807657125, + "prefill_seconds": 0.016352458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114574583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946512666, + "first": 198.062228083, + "last": 202.870853291, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.15677557025132, + "seconds": 4.889105916, + "prefill_seconds": 0.016377709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033394542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.94597725, + "first": 197.980583041, + "last": 202.869653833, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.707620700272, + "seconds": 4.661142209, + "prefill_seconds": 0.016277583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.260805917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946568125, + "first": 198.208940875, + "last": 202.870685583, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.95393048432777, + "seconds": 4.726254375, + "prefill_seconds": 0.016303875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.195915292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946023791, + "first": 198.143698791, + "last": 202.870647958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.861242596841535, + "seconds": 4.823950166, + "prefill_seconds": 0.01634625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098226958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946314666, + "first": 198.045870291, + "last": 202.870816041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.98265410181364, + "seconds": 4.9054825, + "prefill_seconds": 0.016940541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017084917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946113333, + "first": 197.964199458, + "last": 202.869641791, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.327417121993896, + "seconds": 4.693762625, + "prefill_seconds": 0.016323084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228316708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946240291, + "first": 198.176346041, + "last": 202.87076675, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.221206566966686, + "seconds": 4.791323167, + "prefill_seconds": 0.016321584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.130839125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946069333, + "first": 198.078556458, + "last": 202.870587708, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.14007566234809, + "seconds": 4.7100045, + "prefill_seconds": 0.016314625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212103375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946365916, + "first": 198.160016541, + "last": 202.87048725, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.506466084817106, + "seconds": 4.856544708, + "prefill_seconds": 0.016302458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065666708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.94646675, + "first": 198.013189166, + "last": 202.869760958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.401544906694156, + "seconds": 4.77514275, + "prefill_seconds": 0.01628175, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.147076541, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946155583, + "first": 198.094844833, + "last": 202.870727166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.68279198187996, + "seconds": 4.840290167, + "prefill_seconds": 0.016327667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.081921709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946608208, + "first": 198.029519625, + "last": 202.8702055, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.517634027718074, + "seconds": 4.677385667, + "prefill_seconds": 0.016312833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244603667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946196291, + "first": 198.192671041, + "last": 202.870100875, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.33092992421609, + "seconds": 4.87283525, + "prefill_seconds": 0.016299958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049342125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946422875, + "first": 197.9968835, + "last": 202.869671625, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.5834534300851, + "seconds": 4.758931791, + "prefill_seconds": 0.016287667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163394917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.945856791, + "first": 198.111126625, + "last": 202.870542458, + "chunks": 240 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 261.0430596423368, + "ttft_max": 0.019998125000000755, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 261.0291981241995, + "seconds": 0.976902208, + "prefill_seconds": 0.019545625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0196965, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 202.884711916, + "first": 202.904710041, + "last": 203.881560375, + "chunks": 248 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 620.6793785851976, + "ttft_max": 0.07659712499997795, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.68386640173455, + "seconds": 1.586967041, + "prefill_seconds": 0.018783417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076120167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 203.895535375, + "first": 203.9721325, + "last": 205.559074833, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.80574759701076, + "seconds": 1.605735333, + "prefill_seconds": 0.018791208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057380833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 203.895588625, + "first": 203.953345291, + "last": 205.559063166, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.16896340819196, + "seconds": 1.643369875, + "prefill_seconds": 0.019636334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019772292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 203.895639375, + "first": 203.915714291, + "last": 205.559055458, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 156.96882729897197, + "seconds": 1.624526375, + "prefill_seconds": 0.018837042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038587417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 203.895463125, + "first": 203.934550916, + "last": 205.559046166, + "chunks": 249 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 803.2640477075901, + "ttft_max": 0.15217400000000225, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.31499581811627, + "seconds": 2.516902833, + "prefill_seconds": 0.018763333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038594, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.573844833, + "first": 205.613133958, + "last": 208.13001625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.23827830592693, + "seconds": 2.423072708, + "prefill_seconds": 0.018833083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.13241375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.573894208, + "first": 205.707139458, + "last": 208.130425166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.63429028995851, + "seconds": 2.460575542, + "prefill_seconds": 0.018792917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.09493575, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.573939083, + "first": 205.669533, + "last": 208.130074166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.42832582906196, + "seconds": 2.441866208, + "prefill_seconds": 0.018770416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113580292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.57424025, + "first": 205.688305958, + "last": 208.130318666, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.5653613853155, + "seconds": 2.535664333, + "prefill_seconds": 0.019734875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019905292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.574035875, + "first": 205.594377, + "last": 208.129995791, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.84966510914845, + "seconds": 2.479346916, + "prefill_seconds": 0.018774792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076161583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.573984791, + "first": 205.650730083, + "last": 208.130127583, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.07667828584013, + "seconds": 2.498122042, + "prefill_seconds": 0.018817875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057407125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.574106125, + "first": 205.631965333, + "last": 208.130049125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.89272971052287, + "seconds": 2.408097333, + "prefill_seconds": 0.018806458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151230584, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 205.573776875, + "first": 205.725950875, + "last": 208.134015125, + "chunks": 253 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 799.8179957692911, + "ttft_max": 0.30238558299998886, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.7115797791297, + "seconds": 4.93119725, + "prefill_seconds": 0.018804875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188510916, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149626, + "first": 208.340005708, + "last": 213.271452875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.1077184684266, + "seconds": 4.893708792, + "prefill_seconds": 0.018815375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225882042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150235666, + "first": 208.377575416, + "last": 213.271850708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.369354695341165, + "seconds": 5.062602083, + "prefill_seconds": 0.018815208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057438583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149681458, + "first": 208.208409458, + "last": 213.2711705, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.3088641120886, + "seconds": 4.874890792, + "prefill_seconds": 0.0187635, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244614333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150010291, + "first": 208.396334166, + "last": 213.271311916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.90841015481174, + "seconds": 4.91249875, + "prefill_seconds": 0.018757916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2071505, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.14973025, + "first": 208.35875975, + "last": 213.271764833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.18287023633437, + "seconds": 5.081415208, + "prefill_seconds": 0.018837875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038693083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149776708, + "first": 208.189589125, + "last": 213.27097, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.92015856856854, + "seconds": 4.818579666, + "prefill_seconds": 0.018806625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300774125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150295125, + "first": 208.452680708, + "last": 213.271397083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.745412377113176, + "seconds": 5.025084792, + "prefill_seconds": 0.018868375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094998125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150162333, + "first": 208.246045458, + "last": 213.271578916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.556738316460006, + "seconds": 5.043838042, + "prefill_seconds": 0.018774458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07632475, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150066833, + "first": 208.227183375, + "last": 213.270984583, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.99778033383685, + "seconds": 5.100226416, + "prefill_seconds": 0.019620083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019882083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149950208, + "first": 208.170753625, + "last": 213.27092925, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.319368125751765, + "seconds": 4.96888425, + "prefill_seconds": 0.018726959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151211916, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149820791, + "first": 208.302334666, + "last": 213.271914166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.12634641303452, + "seconds": 4.987643708, + "prefill_seconds": 0.01880325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132510542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149907666, + "first": 208.283601583, + "last": 213.271712291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.510550441416676, + "seconds": 4.856166958, + "prefill_seconds": 0.018748667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.263278042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149864916, + "first": 208.415094041, + "last": 213.271539125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.514815451228166, + "seconds": 4.950032292, + "prefill_seconds": 0.018847875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169946291, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150120375, + "first": 208.321178375, + "last": 213.271666708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.713996927962626, + "seconds": 4.837424875, + "prefill_seconds": 0.018780041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.282029875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150433958, + "first": 208.433877958, + "last": 213.271623166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.93547239679412, + "seconds": 5.00633425, + "prefill_seconds": 0.018738667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113766375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149541958, + "first": 208.264785541, + "last": 213.271209083, + "chunks": 252 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 356.51506556960055, + "ttft_max": 0.0180762499999787, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 356.4531651992678, + "seconds": 0.7153815, + "prefill_seconds": 0.0173945, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017618917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 2 + }, + "speculative": { + "rounds": 89, + "drafted": 298, + "accepted": 166, + "acceptance_rate": 0.5570469798657718, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 216.2409795, + "first": 216.25905575, + "last": 216.974313041, + "chunks": 90 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.4558312923031, + "ttft_max": 0.0689538329999948, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.5263986917888, + "seconds": 1.55938125, + "prefill_seconds": 0.017043042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034629166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + }, + "sent": 216.991414125, + "first": 217.026535, + "last": 218.585877125, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.7575112179627, + "seconds": 1.57643375, + "prefill_seconds": 0.017402209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017555875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 216.991469916, + "first": 217.009495541, + "last": 218.585886, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 167.8327708426794, + "seconds": 1.519369541, + "prefill_seconds": 0.016975375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.068522416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 222, + "drafted": 40, + "accepted": 33, + "acceptance_rate": 0.825, + "tokens_per_round": 1.1531531531531531 + }, + "sent": 216.991527583, + "first": 217.060481416, + "last": 218.57982675, + "chunks": 223 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 164.90833606031902, + "seconds": 1.546313583, + "prefill_seconds": 0.016955791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.051565416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 216.991326166, + "first": 217.04349525, + "last": 218.589774083, + "chunks": 225 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 812.625302340236, + "ttft_max": 0.1390859169999885, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.95701780856994, + "seconds": 2.40663625, + "prefill_seconds": 0.0169995, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.10443525, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606616583, + "first": 218.712053958, + "last": 221.118782208, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.25043516050957, + "seconds": 2.493876917, + "prefill_seconds": 0.017096458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017283167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606668125, + "first": 218.624661916, + "last": 221.1184945, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.47972886410659, + "seconds": 2.440664833, + "prefill_seconds": 0.017086292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.070542625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606761375, + "first": 218.678019125, + "last": 221.118684333, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.75777254356842, + "seconds": 2.457647208, + "prefill_seconds": 0.016906459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.053488041, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606812291, + "first": 218.660929833, + "last": 221.118551541, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.04917456095748, + "seconds": 2.47454675, + "prefill_seconds": 0.019356792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036653375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606879208, + "first": 218.644024416, + "last": 221.118540875, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.21514330611976, + "seconds": 2.4236055, + "prefill_seconds": 0.017024, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087507292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606715416, + "first": 218.695048, + "last": 221.118941916, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.74145728307404, + "seconds": 2.388949959, + "prefill_seconds": 0.017080458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138532334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 24, + "accepted": 18, + "acceptance_rate": 0.75, + "tokens_per_round": 1.080168776371308 + }, + "sent": 218.607053583, + "first": 218.7461395, + "last": 221.135044, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.49642829772976, + "seconds": 2.3944465, + "prefill_seconds": 0.016995209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.121442542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 28, + "accepted": 21, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 218.606543458, + "first": 218.729046666, + "last": 221.123450958, + "chunks": 235 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 796.1534147494743, + "ttft_max": 0.27384479200000555, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.26800940189429, + "seconds": 5.072808791, + "prefill_seconds": 0.016912166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.06778725, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152768583, + "first": 221.222254416, + "last": 226.295372291, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.37578979471439, + "seconds": 4.868661666, + "prefill_seconds": 0.017050542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.271505791, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152817833, + "first": 221.426662625, + "last": 226.295984708, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.30002477833443, + "seconds": 4.970757833, + "prefill_seconds": 0.01705575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169690292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152910875, + "first": 221.324447916, + "last": 226.295665875, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.12491136915193, + "seconds": 4.987783708, + "prefill_seconds": 0.016988417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.152699916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152996875, + "first": 221.307384041, + "last": 226.295858625, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.77814942225688, + "seconds": 5.021845083, + "prefill_seconds": 0.017025625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.118747375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153127833, + "first": 221.273364333, + "last": 226.295630291, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.93494626963064, + "seconds": 5.106644125, + "prefill_seconds": 0.017024709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034276375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152865333, + "first": 221.188368333, + "last": 226.295026208, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.01157661760684, + "seconds": 4.902754667, + "prefill_seconds": 0.017041375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.237577584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153558125, + "first": 221.392554083, + "last": 226.295546625, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.76917431684454, + "seconds": 5.123653416, + "prefill_seconds": 0.01709925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017298625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.15331425, + "first": 221.17134425, + "last": 226.294957958, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.95109289172138, + "seconds": 5.004799417, + "prefill_seconds": 0.0170235, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.135711625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.15339475, + "first": 221.290385, + "last": 226.295898458, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.60725271892516, + "seconds": 5.038803458, + "prefill_seconds": 0.01699775, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101790334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153193625, + "first": 221.256347375, + "last": 226.295453666, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.10126144743879, + "seconds": 5.089692208, + "prefill_seconds": 0.016966458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05106175, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152953625, + "first": 221.205344125, + "last": 226.294994416, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.65335031479425, + "seconds": 4.93675625, + "prefill_seconds": 0.017052166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.203563667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153439041, + "first": 221.358512458, + "last": 226.295934875, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.831596943059246, + "seconds": 4.919778958, + "prefill_seconds": 0.017001833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.220550333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153038416, + "first": 221.375508458, + "last": 226.295724916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.47540885983634, + "seconds": 4.95382175, + "prefill_seconds": 0.017001875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.186600666, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153079916, + "first": 221.341449, + "last": 226.29581125, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.437527003547316, + "seconds": 5.055759375, + "prefill_seconds": 0.017075375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.084843917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153274916, + "first": 221.239331875, + "last": 226.295589208, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.19304069563798, + "seconds": 4.885708834, + "prefill_seconds": 0.017049458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.254538875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152695791, + "first": 221.409601916, + "last": 226.295761541, + "chunks": 253 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 356.0100491221774, + "ttft_max": 0.020198124999978972, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 355.9871871193013, + "seconds": 0.716317916, + "prefill_seconds": 0.019727208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01988125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 2 + }, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + }, + "sent": 226.314053625, + "first": 226.33425175, + "last": 227.050523666, + "chunks": 92 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 618.4685158218351, + "ttft_max": 0.07836916599998744, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.27251778546173, + "seconds": 1.591040083, + "prefill_seconds": 0.019357709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077917208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 227.06915325, + "first": 227.147522416, + "last": 228.738690791, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.34876392877254, + "seconds": 1.610369375, + "prefill_seconds": 0.019305875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.058574625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 227.069204916, + "first": 227.128161625, + "last": 228.738491125, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 154.63508038201635, + "seconds": 1.649043667, + "prefill_seconds": 0.019737, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019870542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 227.069281375, + "first": 227.089455666, + "last": 228.738462583, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 156.47366856257813, + "seconds": 1.629667166, + "prefill_seconds": 0.019395083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039290583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 227.069078708, + "first": 227.108854375, + "last": 228.738507458, + "chunks": 250 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 794.2858917207209, + "ttft_max": 0.15848308399998245, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.0587355415921, + "seconds": 2.523285084, + "prefill_seconds": 0.019405292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.061060667, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757089041, + "first": 228.818985875, + "last": 231.342461, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.44137453891526, + "seconds": 2.465164458, + "prefill_seconds": 0.019400916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119085792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757139041, + "first": 228.877125083, + "last": 231.342672666, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.83898746886234, + "seconds": 2.503952625, + "prefill_seconds": 0.019355083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.08043925, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757230458, + "first": 228.838339333, + "last": 231.342510416, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.29101108382824, + "seconds": 2.54260075, + "prefill_seconds": 0.021773917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.041741625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757292416, + "first": 228.799579583, + "last": 231.342144, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.26003787647788, + "seconds": 2.445807667, + "prefill_seconds": 0.019376375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138402208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 3, + "accepted": 1, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757185333, + "first": 228.896502166, + "last": 231.342379708, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.63346743176672, + "seconds": 2.484569667, + "prefill_seconds": 0.019376791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.099703125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757397791, + "first": 228.857733291, + "last": 231.342557458, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.92297573491601, + "seconds": 2.43035425, + "prefill_seconds": 0.019320458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.157714333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 6, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 228.757346791, + "first": 228.915829875, + "last": 231.34614375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 99.43941680009635, + "seconds": 2.564375458, + "prefill_seconds": 0.019845875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.020029833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757016125, + "first": 228.777799, + "last": 231.342134333, + "chunks": 255 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 789.771383093008, + "ttft_max": 0.31042429199999333, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.5585013758026, + "seconds": 5.145434041, + "prefill_seconds": 0.019434375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03891575, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366020625, + "first": 231.406101041, + "last": 236.551513208, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.37207354814985, + "seconds": 5.164863083, + "prefill_seconds": 0.019504125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019657625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366069541, + "first": 231.386665333, + "last": 236.551489041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.314192230717474, + "seconds": 4.874394292, + "prefill_seconds": 0.019387959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.309051, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.367078958, + "first": 231.67750325, + "last": 236.552717291, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.10618255422472, + "seconds": 4.893853042, + "prefill_seconds": 0.019403834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.289837834, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366399625, + "first": 231.658110041, + "last": 236.552315041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.123972476181336, + "seconds": 5.087386083, + "prefill_seconds": 0.019403625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.09677175, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366507041, + "first": 231.464250583, + "last": 236.552484833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.745306071496096, + "seconds": 5.126111791, + "prefill_seconds": 0.019392042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.058307, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366205541, + "first": 231.425497333, + "last": 236.552256583, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.898896549288594, + "seconds": 5.009931792, + "prefill_seconds": 0.019383541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.174048792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366115833, + "first": 231.541814125, + "last": 236.551894041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.2947249526529, + "seconds": 4.971271417, + "prefill_seconds": 0.019353375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212683375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366611583, + "first": 231.580536958, + "last": 236.552053875, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.31405424947672, + "seconds": 5.068166416, + "prefill_seconds": 0.019370708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1160415, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366161416, + "first": 231.483617208, + "last": 236.552527041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.09530423884891, + "seconds": 4.990673875, + "prefill_seconds": 0.01935975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.193347583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366554291, + "first": 231.561176666, + "last": 236.552657791, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.901169968812894, + "seconds": 4.913184041, + "prefill_seconds": 0.0193665, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.27052375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366248708, + "first": 231.638703708, + "last": 236.552156041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.495257347179034, + "seconds": 4.951912334, + "prefill_seconds": 0.019391792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.231982292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366697958, + "first": 231.59992825, + "last": 236.552439875, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.69738391616706, + "seconds": 4.932551334, + "prefill_seconds": 0.019402542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.251379375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366291166, + "first": 231.619342458, + "last": 236.552565583, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.93336384761851, + "seconds": 5.106805958, + "prefill_seconds": 0.019342875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077485708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366456791, + "first": 231.444845416, + "last": 236.551823916, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.70264840649132, + "seconds": 5.029323083, + "prefill_seconds": 0.019391417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.154691708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366335916, + "first": 231.522416166, + "last": 236.55221075, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.50744466196342, + "seconds": 5.048760667, + "prefill_seconds": 0.019405917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.135417708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.36594425, + "first": 231.50303225, + "last": 236.552390583, + "chunks": 255 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 290.8510658622243, + "ttft_max": 0.01669720799998231, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 290.8404504890468, + "seconds": 0.876769375, + "prefill_seconds": 0.016364292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01647625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 239.119373958, + "first": 239.136071166, + "last": 240.012808541, + "chunks": 217 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.741970635724, + "ttft_max": 0.06697054099998923, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 168.2030078750511, + "seconds": 1.516025208, + "prefill_seconds": 0.016288875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066519584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 240.033585875, + "first": 240.100556416, + "last": 241.616544208, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.06071405913494, + "seconds": 1.544886083, + "prefill_seconds": 0.016394459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033796, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 240.033701625, + "first": 240.067897041, + "last": 241.612781625, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.32746514404067, + "seconds": 1.561280583, + "prefill_seconds": 0.017268041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01742925, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 240.033644458, + "first": 240.051507666, + "last": 241.612744708, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.82790179935552, + "seconds": 1.528521292, + "prefill_seconds": 0.016369292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05020025, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 240.033513, + "first": 240.084273625, + "last": 241.6128545, + "chunks": 224 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 830.8281925407863, + "ttft_max": 0.13214558399999987, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.58861198447467, + "seconds": 2.415033167, + "prefill_seconds": 0.016334791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.050086541, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.637057125, + "first": 241.687804708, + "last": 244.102791208, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.92960785183546, + "seconds": 2.340961333, + "prefill_seconds": 0.01633, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131459333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 241.637417541, + "first": 241.769563125, + "last": 244.110488041, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.77019400800891, + "seconds": 2.366145875, + "prefill_seconds": 0.016328375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098779375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.637356291, + "first": 241.736815583, + "last": 244.103246708, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.30634603982891, + "seconds": 2.398727917, + "prefill_seconds": 0.01633, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066277916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.637215333, + "first": 241.704144916, + "last": 244.103204708, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.03474678681029, + "seconds": 2.382403917, + "prefill_seconds": 0.016336375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082566, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.63729, + "first": 241.720486041, + "last": 244.103041875, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.52098436526774, + "seconds": 2.349775958, + "prefill_seconds": 0.016411959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.11517475, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.6371155, + "first": 241.753231291, + "last": 244.102989125, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.1788219264758, + "seconds": 2.447714375, + "prefill_seconds": 0.0172515, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0174245, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.637166333, + "first": 241.65510675, + "last": 244.102778208, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.87980361772534, + "seconds": 2.431354667, + "prefill_seconds": 0.016368875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033824042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.636985416, + "first": 241.67147825, + "last": 244.102807333, + "chunks": 235 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 830.8764623507997, + "ttft_max": 0.2633158749999893, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.2876866740977, + "seconds": 4.876865209, + "prefill_seconds": 0.016316958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0502095, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132334416, + "first": 244.183997083, + "last": 249.060898041, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.0965041857833, + "seconds": 4.713798125, + "prefill_seconds": 0.016320625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.213210958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132382875, + "first": 244.347323291, + "last": 249.061706125, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.91079714136565, + "seconds": 4.730035791, + "prefill_seconds": 0.016310959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196879916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.1327245, + "first": 244.331000041, + "last": 249.061462708, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.46293821308609, + "seconds": 4.860574125, + "prefill_seconds": 0.016295833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066502208, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.13243375, + "first": 244.20029975, + "last": 249.061247041, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.11353376479628, + "seconds": 4.893162708, + "prefill_seconds": 0.016336459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033903458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.1327845, + "first": 244.167669583, + "last": 249.060805666, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.357004782800345, + "seconds": 4.779128833, + "prefill_seconds": 0.016427583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.148012458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132483083, + "first": 244.282021833, + "last": 249.061749333, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.94034400629201, + "seconds": 4.909478458, + "prefill_seconds": 0.017405792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017608125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.13298025, + "first": 244.1513485, + "last": 249.060786416, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.540147133077845, + "seconds": 4.762781084, + "prefill_seconds": 0.016341667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.164288708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132836833, + "first": 244.298358583, + "last": 249.061825791, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.639884162991194, + "seconds": 4.844235584, + "prefill_seconds": 0.016357167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082845584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132530791, + "first": 244.216666208, + "last": 249.060937625, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.99553494433352, + "seconds": 4.811726125, + "prefill_seconds": 0.016302583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.115342583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132880583, + "first": 244.249265375, + "last": 249.061494625, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.815375363532894, + "seconds": 4.8281395, + "prefill_seconds": 0.016290834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0989905, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132577625, + "first": 244.232958875, + "last": 249.061538416, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.66295212256913, + "seconds": 4.664951125, + "prefill_seconds": 0.01633075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.262054, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132931541, + "first": 244.396247416, + "last": 249.061641625, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.28344066227193, + "seconds": 4.697565167, + "prefill_seconds": 0.01627325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.229426625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132624416, + "first": 244.363604333, + "last": 249.061416666, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.47267817352149, + "seconds": 4.681245875, + "prefill_seconds": 0.016305875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2457435, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.133083666, + "first": 244.379900916, + "last": 249.061602625, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.17579718838869, + "seconds": 4.795414709, + "prefill_seconds": 0.01632025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1315775, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.13266925, + "first": 244.265597541, + "last": 249.061794458, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.724359588515654, + "seconds": 4.74645025, + "prefill_seconds": 0.016314709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.180721375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132262041, + "first": 244.314685458, + "last": 249.061325375, + "chunks": 239 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 262.1868024417305, + "ttft_max": 0.019961792000003697, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 262.1737960375258, + "seconds": 0.97263725, + "prefill_seconds": 0.0194825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019646792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 249.082937916, + "first": 249.102899708, + "last": 250.075488708, + "chunks": 248 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 622.4916929902471, + "ttft_max": 0.07671124999998824, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.42673348248232, + "seconds": 1.619801125, + "prefill_seconds": 0.018772542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03857225, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 250.097818916, + "first": 250.136841958, + "last": 251.756618958, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.27650569847583, + "seconds": 1.600989417, + "prefill_seconds": 0.01882125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057384666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 250.09795375, + "first": 250.155666625, + "last": 251.75663625, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.6259703736976, + "seconds": 1.638544, + "prefill_seconds": 0.019652541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01981975, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 250.097881958, + "first": 250.118073, + "last": 251.756569458, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.1673614662196, + "seconds": 1.582206209, + "prefill_seconds": 0.018784208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076146375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 250.097743833, + "first": 250.174455083, + "last": 251.756649083, + "chunks": 249 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 804.0098677360683, + "ttft_max": 0.152253375000015, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.17765117321231, + "seconds": 2.495653375, + "prefill_seconds": 0.018853667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057652791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780259666, + "first": 251.838721958, + "last": 254.334342083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.35011616374786, + "seconds": 2.420500416, + "prefill_seconds": 0.018860667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132804792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780316375, + "first": 251.913991666, + "last": 254.334616541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.95414641308287, + "seconds": 2.476830792, + "prefill_seconds": 0.018830958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0764365, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.78046625, + "first": 251.857560708, + "last": 254.334366208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.00518504450298, + "seconds": 2.405542709, + "prefill_seconds": 0.018781791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1515665, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 251.7805225, + "first": 251.932775875, + "last": 254.3382855, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.65766624393973, + "seconds": 2.533339084, + "prefill_seconds": 0.01976625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019970708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.7803655, + "first": 251.801003208, + "last": 254.33430275, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.53520523380242, + "seconds": 2.439369583, + "prefill_seconds": 0.018769417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113946458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780411666, + "first": 251.895127958, + "last": 254.334824375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.73698945476035, + "seconds": 2.458139583, + "prefill_seconds": 0.018799708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.095197542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780645541, + "first": 251.87635975, + "last": 254.334739291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.41207868563185, + "seconds": 2.514493375, + "prefill_seconds": 0.018852125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038842416, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780173291, + "first": 251.81987025, + "last": 254.334317916, + "chunks": 252 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 797.3471530099499, + "ttft_max": 0.30202095799998574, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.96634395684408, + "seconds": 5.003301791, + "prefill_seconds": 0.018808291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131745375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361831833, + "first": 254.4952495, + "last": 259.498513041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.5454993116723, + "seconds": 4.852937042, + "prefill_seconds": 0.018816667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281690666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.3623815, + "first": 254.645720208, + "last": 259.499229166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.74514911705729, + "seconds": 4.927998167, + "prefill_seconds": 0.018785, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206731125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361883375, + "first": 254.570512166, + "last": 259.498974791, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.74997697292843, + "seconds": 4.834125333, + "prefill_seconds": 0.018753667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300441, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362455208, + "first": 254.664476166, + "last": 259.4988315, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.028713371847566, + "seconds": 5.097072917, + "prefill_seconds": 0.018839959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038308708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361930791, + "first": 254.401221583, + "last": 259.498249, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.844636623049375, + "seconds": 5.115896459, + "prefill_seconds": 0.019437292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019580542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361976666, + "first": 254.382366291, + "last": 259.498230083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.5477064118607, + "seconds": 4.946873833, + "prefill_seconds": 0.018806416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188014375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362214791, + "first": 254.551724583, + "last": 259.498775625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.587413697805275, + "seconds": 5.040779541, + "prefill_seconds": 0.018805667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094318667, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362335333, + "first": 254.457626083, + "last": 259.499060041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.141925990882235, + "seconds": 4.890498292, + "prefill_seconds": 0.01876025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244148834, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362258791, + "first": 254.608091916, + "last": 259.499184583, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.2140262100971, + "seconds": 5.078262375, + "prefill_seconds": 0.018814833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.056949458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362019833, + "first": 254.4200285, + "last": 259.498265416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.40008152954365, + "seconds": 5.059515625, + "prefill_seconds": 0.018778, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075604375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362145166, + "first": 254.438818083, + "last": 259.498341875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.15999607320748, + "seconds": 4.984363166, + "prefill_seconds": 0.018825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150512875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362062166, + "first": 254.514063291, + "last": 259.499334458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.775720261940045, + "seconds": 5.022085333, + "prefill_seconds": 0.018795583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113008042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362104875, + "first": 254.476434333, + "last": 259.499267458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.343333200012474, + "seconds": 4.871680583, + "prefill_seconds": 0.018804125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.262896292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362604041, + "first": 254.626896916, + "last": 259.498732166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.942677792211114, + "seconds": 4.909257875, + "prefill_seconds": 0.018807583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225384334, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362501541, + "first": 254.589343833, + "last": 259.499018333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.35307162482252, + "seconds": 4.965623125, + "prefill_seconds": 0.01884425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169330792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361755, + "first": 254.532916458, + "last": 259.498760125, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 347.10809909124464, + "ttft_max": 0.01903087499999856, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 347.04676560369313, + "seconds": 0.734771291, + "prefill_seconds": 0.0181315, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0184205, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 2 + }, + "speculative": { + "rounds": 89, + "drafted": 305, + "accepted": 166, + "acceptance_rate": 0.5442622950819672, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 262.414576583, + "first": 262.433607458, + "last": 263.168248916, + "chunks": 90 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.8577525210864, + "ttft_max": 0.07029229199997644, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 167.9839637527971, + "seconds": 1.518002042, + "prefill_seconds": 0.017058292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0694655, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 222, + "drafted": 41, + "accepted": 33, + "acceptance_rate": 0.8048780487804879, + "tokens_per_round": 1.1531531531531531 + }, + "sent": 263.223389541, + "first": 263.293681833, + "last": 264.811650125, + "chunks": 223 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.04391455469687, + "seconds": 1.545043334, + "prefill_seconds": 0.017076334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.052446625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 263.223573125, + "first": 263.276616166, + "last": 264.8216225, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.8603572884961, + "seconds": 1.575432084, + "prefill_seconds": 0.017927792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.018190708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 263.223487625, + "first": 263.242327375, + "last": 264.81769975, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.64776206179113, + "seconds": 1.558224792, + "prefill_seconds": 0.017220041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.035387, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + }, + "sent": 263.22323575, + "first": 263.25954125, + "last": 264.817712708, + "chunks": 224 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 816.154892487187, + "ttft_max": 0.13931495899998936, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.6861953253668, + "seconds": 2.412803292, + "prefill_seconds": 0.017041708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087976042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846511833, + "first": 264.935187666, + "last": 267.348084166, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.22283690948855, + "seconds": 2.378224708, + "prefill_seconds": 0.016980625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138756709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 27, + "accepted": 18, + "acceptance_rate": 0.6666666666666666, + "tokens_per_round": 1.080168776371308 + }, + "sent": 264.846802041, + "first": 264.986117, + "last": 267.364310708, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.21882043800122, + "seconds": 2.446774958, + "prefill_seconds": 0.01692975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.053928416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846566291, + "first": 264.90109175, + "last": 267.348025083, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.69546221437821, + "seconds": 2.483069792, + "prefill_seconds": 0.017370417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017606417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846614541, + "first": 264.864785166, + "last": 267.347922041, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.50304534879383, + "seconds": 2.463695625, + "prefill_seconds": 0.019371583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036972417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846660166, + "first": 264.8841655, + "last": 267.348166375, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.43608575659778, + "seconds": 2.395804, + "prefill_seconds": 0.016969625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.104798375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846848041, + "first": 264.95216475, + "last": 267.348208291, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.97931103778156, + "seconds": 2.383638458, + "prefill_seconds": 0.016975375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.121775625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 264.846755708, + "first": 264.969137041, + "last": 267.352739, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.94912645421738, + "seconds": 2.429748666, + "prefill_seconds": 0.017050041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.070941166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846436666, + "first": 264.918142291, + "last": 267.348244166, + "chunks": 234 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 795.3966095266991, + "ttft_max": 0.2762710839999727, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.9112888007362, + "seconds": 5.109064625, + "prefill_seconds": 0.019413458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036750875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.389845166, + "first": 267.427796625, + "last": 272.536823041, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.35051860785666, + "seconds": 4.871011917, + "prefill_seconds": 0.01701525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.273711042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.389894416, + "first": 267.6661655, + "last": 272.537800791, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.276274000501004, + "seconds": 4.97306025, + "prefill_seconds": 0.017041875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.172224, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390209333, + "first": 267.564042208, + "last": 272.537747166, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.58499563729209, + "seconds": 5.0410205, + "prefill_seconds": 0.017008042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.10443325, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390263458, + "first": 267.4959535, + "last": 272.537241583, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.80861373579655, + "seconds": 4.921961458, + "prefill_seconds": 0.017083208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.223340708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.389942, + "first": 267.615203541, + "last": 272.53764725, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.101519986395736, + "seconds": 4.990066833, + "prefill_seconds": 0.017019792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.155221916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390074541, + "first": 267.546992875, + "last": 272.537043083, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.168243030579276, + "seconds": 4.888031208, + "prefill_seconds": 0.017024, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2569055, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.389986416, + "first": 267.649160958, + "last": 272.537388916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.7224988993842, + "seconds": 5.128463083, + "prefill_seconds": 0.017152125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017343333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390161708, + "first": 267.408377666, + "last": 272.53680225, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.629483441703364, + "seconds": 4.939038375, + "prefill_seconds": 0.017034917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206259208, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390523833, + "first": 267.598092291, + "last": 272.537894125, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.92779330146235, + "seconds": 5.007089125, + "prefill_seconds": 0.01701475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138409708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390031375, + "first": 267.529982708, + "last": 272.537136375, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.451851445692355, + "seconds": 4.956089875, + "prefill_seconds": 0.017006417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.189219167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.3903485, + "first": 267.581059916, + "last": 272.537550291, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.24548490747553, + "seconds": 5.075082875, + "prefill_seconds": 0.017020375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07063525, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390606583, + "first": 267.461886208, + "last": 272.537164916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.75611896995513, + "seconds": 5.024024791, + "prefill_seconds": 0.017000916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.12143275, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.39040525, + "first": 267.512956291, + "last": 272.537279708, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.07825053627812, + "seconds": 5.092030917, + "prefill_seconds": 0.017059167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.053735833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390115958, + "first": 267.444854166, + "last": 272.536891, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.41470917348272, + "seconds": 5.058047625, + "prefill_seconds": 0.017059, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087625167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390450208, + "first": 267.47895075, + "last": 272.537340916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.98675938012517, + "seconds": 4.905095125, + "prefill_seconds": 0.0169305, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.240166167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.38977, + "first": 267.632118166, + "last": 272.537432125, + "chunks": 253 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 357.09771979100753, + "ttft_max": 0.02024029099999325, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 357.07005135954194, + "seconds": 0.714145583, + "prefill_seconds": 0.019746083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019916541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 2 + }, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + }, + "sent": 272.563330125, + "first": 272.583570416, + "last": 273.297660666, + "chunks": 92 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 618.4479070805222, + "ttft_max": 0.07851445800002921, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 156.46971991966768, + "seconds": 1.629708292, + "prefill_seconds": 0.01944275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039390166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 273.322900541, + "first": 273.36274, + "last": 274.99240275, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.3548155302584, + "seconds": 1.610307834, + "prefill_seconds": 0.019437709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05881675, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 273.322958916, + "first": 273.382180416, + "last": 274.992586083, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.284594235288, + "seconds": 1.590920208, + "prefill_seconds": 0.019369333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.078157791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 273.323042125, + "first": 273.401556583, + "last": 274.992435333, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 154.6267272904501, + "seconds": 1.64913275, + "prefill_seconds": 0.019795417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019949166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 273.3228215, + "first": 273.343296, + "last": 274.992382833, + "chunks": 250 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 794.1760553413691, + "ttft_max": 0.15879825000001802, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.43158275989286, + "seconds": 2.465397833, + "prefill_seconds": 0.019398875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119088542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018285458, + "first": 275.138415416, + "last": 277.604131583, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.2490729611327, + "seconds": 2.446064917, + "prefill_seconds": 0.01933225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138377875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018488125, + "first": 275.157752625, + "last": 277.603875083, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.83034555279177, + "seconds": 2.504165125, + "prefill_seconds": 0.019374167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.080340625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018336666, + "first": 275.099585541, + "last": 277.603806583, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.91587267057669, + "seconds": 2.430518791, + "prefill_seconds": 0.019425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.157737583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 6, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 275.018384416, + "first": 275.177182666, + "last": 277.607662291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.0498251393604, + "seconds": 2.523507583, + "prefill_seconds": 0.019428208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.061057208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.01854475, + "first": 275.080205, + "last": 277.603754291, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.62420660546744, + "seconds": 2.484793875, + "prefill_seconds": 0.019429167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.099766709, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018601625, + "first": 275.119014791, + "last": 277.603847833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.27760510072491, + "seconds": 2.542940667, + "prefill_seconds": 0.021809916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.041732958, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018434791, + "first": 275.060769916, + "last": 277.603710041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 99.42550415277093, + "seconds": 2.564734292, + "prefill_seconds": 0.019841125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.020004125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.0182115, + "first": 275.038962333, + "last": 277.603698083, + "chunks": 255 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 788.8548028896271, + "ttft_max": 0.31118087500004776, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.06330676926831, + "seconds": 5.093550875, + "prefill_seconds": 0.019405334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.096908417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635270458, + "first": 277.733827041, + "last": 282.82798875, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.83831146504552, + "seconds": 4.919141708, + "prefill_seconds": 0.019410667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.270784958, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635318875, + "first": 277.908384208, + "last": 282.82771525, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.249450247420356, + "seconds": 4.880434125, + "prefill_seconds": 0.019322542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.309351459, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635929833, + "first": 277.947110708, + "last": 282.828041416, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.25550159125926, + "seconds": 5.074071334, + "prefill_seconds": 0.019411417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.116295708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635370625, + "first": 277.753237958, + "last": 282.827383791, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.43216216799861, + "seconds": 4.957987167, + "prefill_seconds": 0.01937825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.232349833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635884291, + "first": 277.869507416, + "last": 282.827847833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.04327510575244, + "seconds": 4.8997685, + "prefill_seconds": 0.019391209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.290074292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635418666, + "first": 277.927773666, + "last": 282.827945541, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.83702243367445, + "seconds": 5.016029417, + "prefill_seconds": 0.019409708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.174288833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.63567375, + "first": 277.811379125, + "last": 282.827585208, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.6411701691176, + "seconds": 5.035428667, + "prefill_seconds": 0.019312167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.154905541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635601791, + "first": 277.791968875, + "last": 282.828079333, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.63429242934714, + "seconds": 4.938578375, + "prefill_seconds": 0.019462375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.251585334, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635465666, + "first": 277.888973083, + "last": 282.827899125, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.23187804161023, + "seconds": 4.97736975, + "prefill_seconds": 0.019391084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212965209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.636000208, + "first": 277.850126291, + "last": 282.828285458, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.68577118614894, + "seconds": 5.132254042, + "prefill_seconds": 0.019380583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.058375583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635724208, + "first": 277.695039291, + "last": 282.827327833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.499774448454225, + "seconds": 5.151538625, + "prefill_seconds": 0.019430542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039145041, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635511791, + "first": 277.675665916, + "last": 282.827238041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.0330804899227, + "seconds": 4.996758917, + "prefill_seconds": 0.019350167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.193647417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.63577375, + "first": 277.830735541, + "last": 282.827683125, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.447223289623366, + "seconds": 5.054787625, + "prefill_seconds": 0.019404584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.135652625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635556958, + "first": 277.772669291, + "last": 282.8278115, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.31379809341647, + "seconds": 5.170966542, + "prefill_seconds": 0.01954675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01973825, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.6358325, + "first": 277.656231, + "last": 282.827159041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.87444922024807, + "seconds": 5.112838417, + "prefill_seconds": 0.019368834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077582083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635199416, + "first": 277.7144185, + "last": 282.827228416, + "chunks": 255 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 290.6181341038439, + "ttft_max": 0.016731250000020736, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 290.60274750059926, + "seconds": 0.877486542, + "prefill_seconds": 0.01636925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016472375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 284.939780666, + "first": 284.956511916, + "last": 285.833952, + "chunks": 217 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.7155973832075, + "ttft_max": 0.0660668329999794, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.32064822925992, + "seconds": 1.56134575, + "prefill_seconds": 0.016514084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016673583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 285.862577625, + "first": 285.879703041, + "last": 287.441005416, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 168.18952852945154, + "seconds": 1.516146708, + "prefill_seconds": 0.016321125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065632417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 285.862633, + "first": 285.928699833, + "last": 287.444802916, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.80816283555382, + "seconds": 1.528702167, + "prefill_seconds": 0.016336208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.04929975, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 285.86270525, + "first": 285.912371416, + "last": 287.441028291, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.0461489711835, + "seconds": 1.545022417, + "prefill_seconds": 0.016329833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.032998208, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 285.862509541, + "first": 285.896034666, + "last": 287.441016958, + "chunks": 224 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 831.0161994582763, + "ttft_max": 0.13210483299997122, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.05744545337508, + "seconds": 2.381898792, + "prefill_seconds": 0.016335083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082185334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473703583, + "first": 287.556703833, + "last": 289.938711166, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.20361821466938, + "seconds": 2.447131917, + "prefill_seconds": 0.016739709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01690125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473754208, + "first": 287.491339583, + "last": 289.938426125, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.5428566352171, + "seconds": 2.349302459, + "prefill_seconds": 0.016337209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114814833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473905833, + "first": 287.5893835, + "last": 289.938776916, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.33209910232699, + "seconds": 2.398146958, + "prefill_seconds": 0.0163225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065811875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473959458, + "first": 287.540367791, + "last": 289.93848275, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.90300174797198, + "seconds": 2.430817, + "prefill_seconds": 0.016336208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033205166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473802458, + "first": 287.507666291, + "last": 289.938498916, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.79540623645326, + "seconds": 2.365592458, + "prefill_seconds": 0.016332541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098416125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.474046916, + "first": 287.57303225, + "last": 289.938839, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.61353180011204, + "seconds": 2.414463333, + "prefill_seconds": 0.016364, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049551, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.47384725, + "first": 287.524044458, + "last": 289.938604958, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.95265378984647, + "seconds": 2.340466167, + "prefill_seconds": 0.0163505, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131109708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 287.473629625, + "first": 287.605734458, + "last": 289.946165375, + "chunks": 237 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 830.5125280653175, + "ttft_max": 0.26301050000000714, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.97328212515975, + "seconds": 4.813747417, + "prefill_seconds": 0.016346375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114878917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974824875, + "first": 290.091412583, + "last": 294.905738875, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.15373023346543, + "seconds": 4.797405542, + "prefill_seconds": 0.016338667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131151584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974872666, + "first": 290.10773875, + "last": 294.905436208, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.88709164860404, + "seconds": 4.732116583, + "prefill_seconds": 0.016302834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196179958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.9752195, + "first": 290.17302875, + "last": 294.905335416, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.335340783530825, + "seconds": 4.781070042, + "prefill_seconds": 0.016332584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.147376375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975468583, + "first": 290.124080791, + "last": 294.905142458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.26651254634462, + "seconds": 4.878840917, + "prefill_seconds": 0.016349958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049839875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974923291, + "first": 290.026090333, + "last": 294.904896666, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.26131145645951, + "seconds": 4.699480959, + "prefill_seconds": 0.01632775, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228727, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974972458, + "first": 290.205723375, + "last": 294.905556041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.44905981674593, + "seconds": 4.683276458, + "prefill_seconds": 0.016322208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.245048459, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975297416, + "first": 290.222047041, + "last": 294.905838458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.0919816449772, + "seconds": 4.895187166, + "prefill_seconds": 0.016356292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033592792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975163875, + "first": 290.009740291, + "last": 294.904886375, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.91876704520609, + "seconds": 4.911518792, + "prefill_seconds": 0.017077209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017252375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975518458, + "first": 289.993361416, + "last": 294.90485675, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.79500372929414, + "seconds": 4.8300025, + "prefill_seconds": 0.016333958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098574666, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975023208, + "first": 290.075069666, + "last": 294.905689416, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.6172965279471, + "seconds": 4.846315125, + "prefill_seconds": 0.0162985, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082317875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975656375, + "first": 290.058719583, + "last": 294.9059905, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.441649342831724, + "seconds": 4.862547292, + "prefill_seconds": 0.016325667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066120916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975071166, + "first": 290.042422, + "last": 294.9055095, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.64040514687041, + "seconds": 4.666876084, + "prefill_seconds": 0.016307709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.261180375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975348291, + "first": 290.238358791, + "last": 294.905790916, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.518162910808236, + "seconds": 4.764737542, + "prefill_seconds": 0.016333209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163652334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.97511825, + "first": 290.140429708, + "last": 294.905645291, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.70165697366044, + "seconds": 4.748456833, + "prefill_seconds": 0.016300625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.17993075, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975405166, + "first": 290.15672825, + "last": 294.905898458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.0732705539128, + "seconds": 4.7158235, + "prefill_seconds": 0.0163535, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212464833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974755416, + "first": 290.189391083, + "last": 294.905949541, + "chunks": 240 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 262.3964081747341, + "ttft_max": 0.0202437500000201, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 262.3845845455222, + "seconds": 0.971855875, + "prefill_seconds": 0.019778708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01992125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 294.935785083, + "first": 294.956028833, + "last": 295.927840916, + "chunks": 248 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 622.917485317143, + "ttft_max": 0.07658683299996483, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.28302862963503, + "seconds": 1.5810715, + "prefill_seconds": 0.018786167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076180834, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 295.956686583, + "first": 296.033273416, + "last": 297.614340125, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.38804162765362, + "seconds": 1.599869083, + "prefill_seconds": 0.018792791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057388042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 295.956737458, + "first": 296.01448975, + "last": 297.614322583, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.53843007753804, + "seconds": 1.618652667, + "prefill_seconds": 0.018812834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03856325, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 295.956797541, + "first": 295.995684208, + "last": 297.614334, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.72805176609543, + "seconds": 1.637469917, + "prefill_seconds": 0.019629625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01979225, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 295.9566095, + "first": 295.976884083, + "last": 297.614306875, + "chunks": 249 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 804.202708162487, + "ttft_max": 0.15177995799996324, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.68020664965388, + "seconds": 2.532771917, + "prefill_seconds": 0.019620959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019774875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.64415175, + "first": 297.664605583, + "last": 300.197336166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.43305010971675, + "seconds": 2.5139735, + "prefill_seconds": 0.018797042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038582458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644201708, + "first": 297.683404083, + "last": 300.197378416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.55656664589235, + "seconds": 2.438871208, + "prefill_seconds": 0.018806958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113656292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644494291, + "first": 297.758623166, + "last": 300.197750125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.36886563708484, + "seconds": 2.420069709, + "prefill_seconds": 0.01880625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132486167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.64434525, + "first": 297.777433, + "last": 300.197624833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.02447525034125, + "seconds": 2.405105042, + "prefill_seconds": 0.018772417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151229541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 297.644441958, + "first": 297.796221916, + "last": 300.201279458, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.19631740884425, + "seconds": 2.495197542, + "prefill_seconds": 0.018787666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0573445, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644294041, + "first": 297.702194083, + "last": 300.197352166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.97124018309296, + "seconds": 2.476419625, + "prefill_seconds": 0.018807417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076114708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644248791, + "first": 297.720997833, + "last": 300.197710375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.75857642910877, + "seconds": 2.457628167, + "prefill_seconds": 0.018811125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094907375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644084541, + "first": 297.739818708, + "last": 300.197828625, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 800.4553963396994, + "ttft_max": 0.30195220800004563, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.759322934436305, + "seconds": 4.833269, + "prefill_seconds": 0.018797542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281478292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231652083, + "first": 300.51546325, + "last": 305.348800291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.22411279350947, + "seconds": 5.0772425, + "prefill_seconds": 0.018789584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03817825, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231700833, + "first": 300.271134916, + "last": 305.348341833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.039204735749585, + "seconds": 5.09600425, + "prefill_seconds": 0.019426541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019594541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232012, + "first": 300.252341125, + "last": 305.348313, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.95107702209289, + "seconds": 4.908464167, + "prefill_seconds": 0.018742917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206534083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232607625, + "first": 300.440241875, + "last": 305.349147916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.16930216100543, + "seconds": 4.983456667, + "prefill_seconds": 0.0187925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1318505, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231747833, + "first": 300.365159166, + "last": 305.349047041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.15083463192746, + "seconds": 4.889662875, + "prefill_seconds": 0.018801, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225291708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.23179275, + "first": 300.459042875, + "last": 305.348887208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.353013017002034, + "seconds": 4.870779833, + "prefill_seconds": 0.018853834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244087125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232240375, + "first": 300.477911166, + "last": 305.349439625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.55742295669859, + "seconds": 4.945941542, + "prefill_seconds": 0.018782875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232067208, + "first": 300.402732291, + "last": 305.349306333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.36257397740955, + "seconds": 4.964704458, + "prefill_seconds": 0.018785167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150635792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232163083, + "first": 300.38394475, + "last": 305.349396666, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.787357994357095, + "seconds": 5.020934541, + "prefill_seconds": 0.01880575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094437167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.23196425, + "first": 300.327546375, + "last": 305.348466166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.41054903674682, + "seconds": 5.058465041, + "prefill_seconds": 0.018778792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05694, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231836916, + "first": 300.289925708, + "last": 305.348357458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.97766116066705, + "seconds": 5.002191042, + "prefill_seconds": 0.018801375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113120959, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231921875, + "first": 300.34635125, + "last": 305.348693541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.59845446028725, + "seconds": 5.039679625, + "prefill_seconds": 0.018817625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075707583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232111583, + "first": 300.308746625, + "last": 305.348481875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.55521388941559, + "seconds": 4.85204, + "prefill_seconds": 0.018757833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.262793375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231878, + "first": 300.49667725, + "last": 305.349356333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.96454363918808, + "seconds": 4.814541625, + "prefill_seconds": 0.018778625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.3002615, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232298208, + "first": 300.534250416, + "last": 305.348984708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.754304328772875, + "seconds": 4.927126416, + "prefill_seconds": 0.018756542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.187937709, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231581625, + "first": 300.421495166, + "last": 305.3491995, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 290.11219414456707, + "ttft_max": 0.016669167000031848, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 290.0973561038901, + "seconds": 0.87901525, + "prefill_seconds": 0.016310084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016434416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 308.221080041, + "first": 308.237749208, + "last": 309.1167195, + "chunks": 217 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.566298308141, + "ttft_max": 0.0668896669999981, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.01353390530363, + "seconds": 1.545327792, + "prefill_seconds": 0.016315083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033670666, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 309.150666166, + "first": 309.184809333, + "last": 310.730107083, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.28918201965183, + "seconds": 1.561646625, + "prefill_seconds": 0.017215208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017364709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 309.150722833, + "first": 309.168494708, + "last": 310.730092875, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.7782701067951, + "seconds": 1.528976166, + "prefill_seconds": 0.01635025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049985708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 309.150815208, + "first": 309.201149708, + "last": 310.730115875, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 168.14815632964886, + "seconds": 1.51651975, + "prefill_seconds": 0.0163105, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066292083, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 309.150580458, + "first": 309.217470125, + "last": 310.733953208, + "chunks": 225 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 830.9803317342685, + "ttft_max": 0.1320373749999817, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.89801571795324, + "seconds": 2.430932542, + "prefill_seconds": 0.016297417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033722542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.766304708, + "first": 310.8007125, + "last": 313.231627958, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.78214943631772, + "seconds": 2.365883417, + "prefill_seconds": 0.016259416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098723833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.76635525, + "first": 310.865833791, + "last": 313.232120041, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.31822267581995, + "seconds": 2.398459959, + "prefill_seconds": 0.016258, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066167792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.766503166, + "first": 310.833245791, + "last": 313.232074625, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.043218122123, + "seconds": 2.382215375, + "prefill_seconds": 0.016325084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082435417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.76645175, + "first": 310.849569666, + "last": 313.231988041, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.93092430449789, + "seconds": 2.340933042, + "prefill_seconds": 0.016338959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131257625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 310.766405625, + "first": 310.898443, + "last": 313.239348458, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.52330130464998, + "seconds": 2.349725791, + "prefill_seconds": 0.016277667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114925291, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.766615375, + "first": 310.882109, + "last": 313.23203375, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.19956955362132, + "seconds": 2.447227, + "prefill_seconds": 0.017238375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017401583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.76654975, + "first": 310.784416708, + "last": 313.231598375, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.6030839121993, + "seconds": 2.414702209, + "prefill_seconds": 0.016262292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049953875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.766232208, + "first": 310.816983458, + "last": 313.231833333, + "chunks": 235 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 831.0121583537492, + "ttft_max": 0.2618807499999889, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.95027045081604, + "seconds": 4.908540375, + "prefill_seconds": 0.016921542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017110583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272857333, + "first": 313.291140416, + "last": 318.199646541, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.12312565623399, + "seconds": 4.89226225, + "prefill_seconds": 0.016278375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033401167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272908541, + "first": 313.307429333, + "last": 318.199663041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.66534460310047, + "seconds": 4.664746959, + "prefill_seconds": 0.016233084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.260031625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273462375, + "first": 313.535343125, + "last": 318.200459708, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.730871070835924, + "seconds": 4.745875042, + "prefill_seconds": 0.016297833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.179193292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272953875, + "first": 313.45405375, + "last": 318.200762041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.47625756790819, + "seconds": 4.680938291, + "prefill_seconds": 0.016249583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.243827667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273226625, + "first": 313.519107333, + "last": 318.200277708, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.184599992366444, + "seconds": 4.794621, + "prefill_seconds": 0.01626375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.130646833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273403583, + "first": 313.405226166, + "last": 318.200390541, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.36413897051653, + "seconds": 4.778489917, + "prefill_seconds": 0.016247625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.146778292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.27350225, + "first": 313.421475666, + "last": 318.200350208, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.47296465536849, + "seconds": 4.859645375, + "prefill_seconds": 0.016370417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065873542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273087, + "first": 313.340081291, + "last": 318.199738166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.00491396626079, + "seconds": 4.810874708, + "prefill_seconds": 0.016333583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114492333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272999583, + "first": 313.388962916, + "last": 318.200301416, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.64890537069318, + "seconds": 4.843405541, + "prefill_seconds": 0.016271167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082010875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273128125, + "first": 313.35635825, + "last": 318.200141375, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.915120953167545, + "seconds": 4.729656458, + "prefill_seconds": 0.016279875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.19541025, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.27316975, + "first": 313.470335083, + "last": 318.200551041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.82363565611979, + "seconds": 4.8273845, + "prefill_seconds": 0.016271416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098151458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273042916, + "first": 313.372629041, + "last": 318.200660833, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.10040781099566, + "seconds": 4.713458, + "prefill_seconds": 0.016259792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.211572583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273303208, + "first": 313.486597958, + "last": 318.200508166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.54590708472722, + "seconds": 4.76226875, + "prefill_seconds": 0.016263292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163035166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273360625, + "first": 313.4377435, + "last": 318.20042525, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.28749844480688, + "seconds": 4.697214042, + "prefill_seconds": 0.016251375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2276415, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.2736095, + "first": 313.502852125, + "last": 318.200815875, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.296831998913866, + "seconds": 4.876012375, + "prefill_seconds": 0.016277333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049526583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272782291, + "first": 313.323708, + "last": 318.19969325, + "chunks": 240 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 262.21991974165917, + "ttft_max": 0.019898124999997435, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 262.2083813625824, + "seconds": 0.972508959, + "prefill_seconds": 0.019450166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019591042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 318.232904875, + "first": 318.252803, + "last": 319.225269166, + "chunks": 248 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 622.948506893192, + "ttft_max": 0.07660004199999548, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.54933154086103, + "seconds": 1.618540666, + "prefill_seconds": 0.018821792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038648084, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 319.259709291, + "first": 319.298847208, + "last": 320.9173675, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.28251001913515, + "seconds": 1.581076584, + "prefill_seconds": 0.018785584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076145417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 319.259760166, + "first": 319.336360208, + "last": 320.917396375, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.73814530505885, + "seconds": 1.637363791, + "prefill_seconds": 0.019692083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019849542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 319.259818458, + "first": 319.280021875, + "last": 320.917341833, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.39187730854874, + "seconds": 1.599830583, + "prefill_seconds": 0.018723917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057366708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 319.259639291, + "first": 319.317571416, + "last": 320.917359416, + "chunks": 249 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 804.3912665584089, + "ttft_max": 0.1521238329999619, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.9992905473494, + "seconds": 2.475745208, + "prefill_seconds": 0.018804291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076227416, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951588666, + "first": 321.028649208, + "last": 323.504430833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.0499026134387, + "seconds": 2.404528375, + "prefill_seconds": 0.018741125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150904375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 320.951646583, + "first": 321.103770416, + "last": 323.508267916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.78672128658181, + "seconds": 2.456961708, + "prefill_seconds": 0.018798958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094958666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951808583, + "first": 321.04744825, + "last": 323.504369916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.22434666240264, + "seconds": 2.494513375, + "prefill_seconds": 0.01880375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057425166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951908666, + "first": 321.009842, + "last": 323.504324, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.58395403722076, + "seconds": 2.438232541, + "prefill_seconds": 0.018791667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1136865, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951854583, + "first": 321.066244125, + "last": 323.504529625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.45915505061564, + "seconds": 2.513326667, + "prefill_seconds": 0.018847541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03865225, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951702833, + "first": 320.991028, + "last": 323.504351708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.7042236486789, + "seconds": 2.532167875, + "prefill_seconds": 0.019715959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019900375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.9517515, + "first": 320.972188666, + "last": 323.504311291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.39538069612306, + "seconds": 2.419460875, + "prefill_seconds": 0.018780292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132466625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951514833, + "first": 321.085027083, + "last": 323.50478975, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 801.2450260525325, + "ttft_max": 0.3025994579999747, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.08908039467293, + "seconds": 5.090929959, + "prefill_seconds": 0.019595375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019816333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542316958, + "first": 323.563395791, + "last": 328.654280083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.27460697129549, + "seconds": 5.072143083, + "prefill_seconds": 0.0188055, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038602, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542368708, + "first": 323.582208666, + "last": 328.654311625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.648919005256076, + "seconds": 5.034658291, + "prefill_seconds": 0.018770458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075997792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.54271425, + "first": 323.619719958, + "last": 328.654323666, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.809909127089995, + "seconds": 4.82863925, + "prefill_seconds": 0.018726458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281503583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542420375, + "first": 323.826029125, + "last": 328.655109291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.40358953285565, + "seconds": 4.866078875, + "prefill_seconds": 0.018754083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244145875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542962541, + "first": 323.788552166, + "last": 328.654811041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.837700640704554, + "seconds": 5.0159625, + "prefill_seconds": 0.018772625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094714792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542471, + "first": 323.638498125, + "last": 328.654887041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.605931519919196, + "seconds": 4.847362125, + "prefill_seconds": 0.018756417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.262877833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542769458, + "first": 323.80731225, + "last": 328.654937416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.20315081329055, + "seconds": 4.884762625, + "prefill_seconds": 0.018713334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225421875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542519916, + "first": 323.769792833, + "last": 328.654603416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.46094673811927, + "seconds": 5.053412916, + "prefill_seconds": 0.018739, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057202375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.543029958, + "first": 323.600936041, + "last": 328.655159375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.00380915298769, + "seconds": 4.903486959, + "prefill_seconds": 0.018685375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206730875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542568375, + "first": 323.751071833, + "last": 328.654986166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.41379407069505, + "seconds": 4.959758458, + "prefill_seconds": 0.018779375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.15092825, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542815041, + "first": 323.69485825, + "last": 328.655296208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.60879976933287, + "seconds": 4.941017833, + "prefill_seconds": 0.018750542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169453166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542615541, + "first": 323.713606541, + "last": 328.655200291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.805692932797264, + "seconds": 4.922238958, + "prefill_seconds": 0.01877275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188166209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542900833, + "first": 323.732385958, + "last": 328.654701541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.2207366147053, + "seconds": 4.978452417, + "prefill_seconds": 0.018818875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.13210325, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542660416, + "first": 323.676052791, + "last": 328.655355208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.02788546370878, + "seconds": 4.997267625, + "prefill_seconds": 0.018750125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113471917, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542856125, + "first": 323.657248791, + "last": 328.654717333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.01680247008635, + "seconds": 4.809795916, + "prefill_seconds": 0.018807708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300145291, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542239333, + "first": 323.844838791, + "last": 328.655471083, + "chunks": 252 + } + ] + } + ], + "performance-qwen": [ + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 211.92194615871247, + "ttft_max": 0.03147329199998694, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 211.73019910553273, + "seconds": 1.204362916, + "prefill_seconds": 0.030946833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031178042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 220.468364583, + "first": 220.499837875, + "last": 221.703111083, + "chunks": 44 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 379.28424450667757, + "ttft_max": 0.15881983299999547, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.82656232856375, + "seconds": 2.606654, + "prefill_seconds": 0.117248833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.117549, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 221.705125833, + "first": 221.823199625, + "last": 224.429784583, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 99.3859749337853, + "seconds": 2.565754375, + "prefill_seconds": 0.123503708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158455209, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 221.705284458, + "first": 221.864104291, + "last": 224.429873916, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 96.30150148406732, + "seconds": 2.647933792, + "prefill_seconds": 0.075959041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076291, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 221.705184541, + "first": 221.78189875, + "last": 224.429773291, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.82365861929128, + "seconds": 2.689202291, + "prefill_seconds": 0.034662459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034964041, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 221.705046708, + "first": 221.740597958, + "last": 224.429743875, + "chunks": 68 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 140.9551661001172, + "ttft_max": 0.03612595900000315, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 140.8800278113749, + "seconds": 1.81005075, + "prefill_seconds": 0.035580708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03579525, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 224.430958291, + "first": 224.46708425, + "last": 226.276170125, + "chunks": 66 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 314.3405067152877, + "ttft_max": 0.15906683299999713, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.5539987986341, + "seconds": 3.165578417, + "prefill_seconds": 0.079156209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119072042, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 226.278272291, + "first": 226.397800083, + "last": 229.563340541, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.57684958648021, + "seconds": 3.125886833, + "prefill_seconds": 0.118852042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158741084, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 226.278422708, + "first": 226.437489541, + "last": 229.563531708, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 78.59021380751824, + "seconds": 3.244678792, + "prefill_seconds": 0.039655625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03991425, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 226.278334458, + "first": 226.318642875, + "last": 229.563267041, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.5624005567496, + "seconds": 3.2050315, + "prefill_seconds": 0.079313625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.079552834, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 226.278194875, + "first": 226.358296083, + "last": 229.563297333, + "chunks": 90 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 212.45570139283868, + "ttft_max": 0.03133441599999287, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 212.2655129411728, + "seconds": 1.201325625, + "prefill_seconds": 0.030853583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03105075, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 240.36544375, + "first": 240.396778166, + "last": 241.597028375, + "chunks": 44 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 377.1968654072174, + "ttft_max": 0.15561083300002565, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.27497352907643, + "seconds": 2.621434792, + "prefill_seconds": 0.082676, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113711334, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 241.599939541, + "first": 241.714227666, + "last": 244.335596875, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.30269209545193, + "seconds": 2.704058541, + "prefill_seconds": 0.030840083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031085166, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 1 + }, + "speculative": { + "rounds": 67, + "drafted": 251, + "accepted": 188, + "acceptance_rate": 0.749003984063745, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 241.60006575, + "first": 241.631526791, + "last": 244.335539708, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 98.83527473590149, + "seconds": 2.5800505, + "prefill_seconds": 0.12406575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.155116875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 241.599997125, + "first": 241.755607958, + "last": 244.335685, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 95.77020109254082, + "seconds": 2.662623625, + "prefill_seconds": 0.041446, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07251225, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 241.599864458, + "first": 241.672987208, + "last": 244.335552958, + "chunks": 68 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 141.19484385798825, + "ttft_max": 0.036089042000014615, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 141.11870149437556, + "seconds": 1.806989416, + "prefill_seconds": 0.035520292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0357655, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 244.337321541, + "first": 244.373410583, + "last": 246.179425541, + "chunks": 66 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 316.415283652713, + "ttft_max": 0.15867833299998324, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.10471251890549, + "seconds": 3.223575333, + "prefill_seconds": 0.039642875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039891958, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 246.182093458, + "first": 246.222425833, + "last": 249.445940416, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 82.12139861437048, + "seconds": 3.105159, + "prefill_seconds": 0.118494625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.15834425, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 246.182233208, + "first": 246.340911541, + "last": 249.4460375, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.08921362016, + "seconds": 3.144684584, + "prefill_seconds": 0.118604958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.118837791, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 246.182148166, + "first": 246.301384375, + "last": 249.446024291, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.08463427060869, + "seconds": 3.184131417, + "prefill_seconds": 0.079125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07937475, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 246.18201175, + "first": 246.261895833, + "last": 249.445976166, + "chunks": 89 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 211.59088653158616, + "ttft_max": 0.0313943330000086, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 211.40970570965607, + "seconds": 1.206188709, + "prefill_seconds": 0.030824792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031058833, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 254.012542333, + "first": 254.043936666, + "last": 255.249092541, + "chunks": 44 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 377.49507004207, + "ttft_max": 0.16005191600001467, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 95.84680597070513, + "seconds": 2.660495542, + "prefill_seconds": 0.076353833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076651, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 255.252503833, + "first": 255.329673833, + "last": 257.9901095, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.3572455490473, + "seconds": 2.619219541, + "prefill_seconds": 0.11765925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.117936625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 255.252631541, + "first": 255.370974916, + "last": 257.990136916, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.37434527797964, + "seconds": 2.7020055, + "prefill_seconds": 0.034829917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.035106667, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 255.252561583, + "first": 255.288130666, + "last": 257.990094125, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 98.92393069777353, + "seconds": 2.57773825, + "prefill_seconds": 0.12433075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.159474, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 255.252426125, + "first": 255.412478041, + "last": 257.990152708, + "chunks": 68 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 140.8142774005683, + "ttft_max": 0.03615183299996261, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 140.73994300998172, + "seconds": 1.811852375, + "prefill_seconds": 0.035574334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.035819583, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 257.992488125, + "first": 258.028639958, + "last": 259.839535875, + "chunks": 66 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 315.9810922490545, + "ttft_max": 0.15894150000002583, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.98834255321776, + "seconds": 3.148601292, + "prefill_seconds": 0.079248459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119086042, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 259.842767708, + "first": 259.962342875, + "last": 263.11114075, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.98105127825862, + "seconds": 3.188255167, + "prefill_seconds": 0.079208459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.079422916, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 259.842903583, + "first": 259.922682583, + "last": 263.110874791, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 82.01407486572037, + "seconds": 3.109222416, + "prefill_seconds": 0.118653292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158542541, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 259.842822541, + "first": 260.001764041, + "last": 263.111057041, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.00105489643892, + "seconds": 3.227805, + "prefill_seconds": 0.039626042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039891209, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 259.842679041, + "first": 259.8830995, + "last": 263.110842166, + "chunks": 89 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 212.29582007221308, + "ttft_max": 0.03140174999998635, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 212.119392935892, + "seconds": 1.202153167, + "prefill_seconds": 0.030904875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031109208, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 267.675353583, + "first": 267.706755333, + "last": 268.907909458, + "chunks": 44 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 379.89684590178143, + "ttft_max": 0.16034129099995198, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 99.56640459004593, + "seconds": 2.561104833, + "prefill_seconds": 0.159479333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.159775416, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 268.911875375, + "first": 269.072216666, + "last": 271.633347125, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.98995228321633, + "seconds": 2.602307625, + "prefill_seconds": 0.118263, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.118519333, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 268.911995541, + "first": 269.031000583, + "last": 271.633265916, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.97800686107192, + "seconds": 2.684832083, + "prefill_seconds": 0.035676625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.035969, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 268.911934375, + "first": 268.94840775, + "last": 271.633184708, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 96.45839907288453, + "seconds": 2.643626708, + "prefill_seconds": 0.076915875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077211458, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 268.91179475, + "first": 268.989663083, + "last": 271.633212541, + "chunks": 68 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 141.11723064905576, + "ttft_max": 0.03655066699997178, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 141.04172122135094, + "seconds": 1.807975667, + "prefill_seconds": 0.036057083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036274583, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 271.636266583, + "first": 271.67281725, + "last": 273.4798255, + "chunks": 66 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 314.2222738472332, + "ttft_max": 0.1598498339999992, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.52107452016692, + "seconds": 3.166872791, + "prefill_seconds": 0.119563166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119831125, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 273.483816583, + "first": 273.604093333, + "last": 276.770985458, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.54333064693361, + "seconds": 3.12717175, + "prefill_seconds": 0.118842625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.159505875, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 273.483945416, + "first": 273.64379525, + "last": 276.771061, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 78.55952839607261, + "seconds": 3.245946166, + "prefill_seconds": 0.040431458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.040645833, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 273.483870916, + "first": 273.524951208, + "last": 276.770851416, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.52676274383238, + "seconds": 3.20646775, + "prefill_seconds": 0.079922917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.080200708, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 273.483737416, + "first": 273.564455, + "last": 276.770873291, + "chunks": 90 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 212.0302205426082, + "ttft_max": 0.03127620800000841, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 211.84478174379822, + "seconds": 1.2037115, + "prefill_seconds": 0.03071525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.030954333, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 281.280602625, + "first": 281.311878833, + "last": 282.514537583, + "chunks": 44 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 379.1363966258468, + "ttft_max": 0.16037083300000177, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 99.36423934604692, + "seconds": 2.566315625, + "prefill_seconds": 0.159496, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.159762459, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 282.519196958, + "first": 282.679567791, + "last": 285.24584725, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.78525189785174, + "seconds": 2.607755209, + "prefill_seconds": 0.11804275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.118341166, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 282.519314875, + "first": 282.638117083, + "last": 285.245836083, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.78495060310176, + "seconds": 2.6903005, + "prefill_seconds": 0.035455125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0357835, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 282.519259916, + "first": 282.555522583, + "last": 285.245777125, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 96.25564895196716, + "seconds": 2.649195167, + "prefill_seconds": 0.076599458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0768925, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 282.51910575, + "first": 282.596677875, + "last": 285.245823958, + "chunks": 68 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 141.19561917656972, + "ttft_max": 0.03684129200001962, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 141.1207905160334, + "seconds": 1.806962667, + "prefill_seconds": 0.036193416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03647625, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 285.249543708, + "first": 285.286385, + "last": 287.092390041, + "chunks": 66 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 314.095888200934, + "ttft_max": 0.1594045840000149, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.49345545633638, + "seconds": 3.129085625, + "prefill_seconds": 0.158699542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158901208, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 287.097095166, + "first": 287.25649975, + "last": 290.385539375, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.4908501939986, + "seconds": 3.207916375, + "prefill_seconds": 0.079819542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.080064833, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 287.09728275, + "first": 287.177629958, + "last": 290.385496, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 78.52426526343883, + "seconds": 3.247403833, + "prefill_seconds": 0.040318292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.040579875, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 287.097213666, + "first": 287.138123416, + "last": 290.385478208, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.47927459453535, + "seconds": 3.168517625, + "prefill_seconds": 0.119257542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119520917, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 287.097006458, + "first": 287.217065125, + "last": 290.385527291, + "chunks": 90 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 212.46021523124688, + "ttft_max": 0.031284874999983, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 212.2716090156747, + "seconds": 1.201291125, + "prefill_seconds": 0.03071125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03097, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 294.944889416, + "first": 294.976174291, + "last": 296.176399, + "chunks": 44 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 379.1938092247709, + "ttft_max": 0.1560188750000293, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.80090021819542, + "seconds": 2.689847875, + "prefill_seconds": 0.031562542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031844084, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 1 + }, + "speculative": { + "rounds": 67, + "drafted": 251, + "accepted": 188, + "acceptance_rate": 0.749003984063745, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 296.181794916, + "first": 296.214057333, + "last": 298.903864, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.7997297917729, + "seconds": 2.607369167, + "prefill_seconds": 0.082535875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114337917, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 296.1819095, + "first": 296.296626625, + "last": 298.903974666, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 99.37142863907908, + "seconds": 2.566129958, + "prefill_seconds": 0.123787125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.155574042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 296.181857875, + "first": 296.33787675, + "last": 298.903949958, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 96.2784902404724, + "seconds": 2.648566667, + "prefill_seconds": 0.041288375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.073060333, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 296.181708541, + "first": 296.255363, + "last": 298.90387875, + "chunks": 68 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 141.31967120959874, + "ttft_max": 0.03666970799997671, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 141.24636394193516, + "seconds": 1.805356208, + "prefill_seconds": 0.036061958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036302625, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 298.908591208, + "first": 298.945260916, + "last": 300.749680625, + "chunks": 66 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 316.7578775374469, + "ttft_max": 0.15931695800003354, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 82.21321081821357, + "seconds": 3.101691291, + "prefill_seconds": 0.118398959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158899334, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 300.755190833, + "first": 300.914507791, + "last": 304.016232083, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.17450267878608, + "seconds": 3.180562292, + "prefill_seconds": 0.079673916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.079946708, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 300.755306, + "first": 300.835610666, + "last": 304.016114666, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.17991883253512, + "seconds": 3.141170916, + "prefill_seconds": 0.119087542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119339375, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 300.755252541, + "first": 300.875013291, + "last": 304.016133875, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.19185080228227, + "seconds": 3.220028291, + "prefill_seconds": 0.040182916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.040450042, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 300.755108875, + "first": 300.796106958, + "last": 304.016091958, + "chunks": 89 + } + ] + } + ], + "cold-prefill": [ + { + "round": 0, + "arm": "main-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 9.475e-05, + "prefill_seconds": 0.152224375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.152786125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.15443183299999852 + }, + { + "round": 0, + "arm": "main-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.8958e-05, + "prefill_seconds": 0.602027959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.60268975, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.6041695000000011 + }, + { + "round": 0, + "arm": "main-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6042e-05, + "prefill_seconds": 2.92371825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.924125542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9251302499999987 + }, + { + "round": 0, + "arm": "main-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6083e-05, + "prefill_seconds": 7.367716917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.368493834, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.370146667 + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.9416e-05, + "prefill_seconds": 0.142128125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.142321375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.1428019169999999 + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4333e-05, + "prefill_seconds": 0.596952333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.597156042, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.5976716660000001 + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4583e-05, + "prefill_seconds": 2.92241675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.922817417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9238302500000017 + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.5083e-05, + "prefill_seconds": 7.322124959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.322879042, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.324568999999997 + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 5.0375e-05, + "prefill_seconds": 0.162896375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.16343125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.16452287500000295 + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.7708e-05, + "prefill_seconds": 0.632174167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.632618458, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.6336222919999983 + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.8084e-05, + "prefill_seconds": 3.046362584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 3.046803042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 3.0479570839999965 + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6958e-05, + "prefill_seconds": 7.564455625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.565224041, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.566995167000002 + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.5834e-05, + "prefill_seconds": 0.142068208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.142249625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.14270199999999988 + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.5542e-05, + "prefill_seconds": 0.596721958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.596929459, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.5973893749999988 + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4916e-05, + "prefill_seconds": 2.922063542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.922471709, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.923471584000005 + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.1833e-05, + "prefill_seconds": 7.322574708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.323297667, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.325058874999996 + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.8791e-05, + "prefill_seconds": 0.16274975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163254084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.16436254200000633 + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 2.5375e-05, + "prefill_seconds": 0.632193375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.632682667, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.6336696670000066 + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.8917e-05, + "prefill_seconds": 3.045708417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 3.046133125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 3.0472295000000003 + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.0959e-05, + "prefill_seconds": 7.564470417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.565208167, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.566989333999999 + }, + { + "round": 1, + "arm": "main-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 6.125e-05, + "prefill_seconds": 0.152161958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.152673334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.15385920900000372 + }, + { + "round": 1, + "arm": "main-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.5958e-05, + "prefill_seconds": 0.60266275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.603419666, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.6049087920000034 + }, + { + "round": 1, + "arm": "main-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.7e-05, + "prefill_seconds": 2.924098958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.924542125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9256166250000035 + }, + { + "round": 1, + "arm": "main-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.725e-05, + "prefill_seconds": 7.326498708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.32727875, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.329111124999997 + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 6.3167e-05, + "prefill_seconds": 0.16316275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163665083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.16489629199999456 + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.0083e-05, + "prefill_seconds": 0.632195333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.632729583, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.633867084000002 + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.2125e-05, + "prefill_seconds": 3.046123, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 3.04655725, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 3.0476708340000016 + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.1291e-05, + "prefill_seconds": 7.564734041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.565487958, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.567298958999999 + }, + { + "round": 2, + "arm": "main-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.9333e-05, + "prefill_seconds": 0.142121708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.142318875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.14281366699999865 + }, + { + "round": 2, + "arm": "main-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.575e-05, + "prefill_seconds": 0.597101375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.597292042, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.5977911669999969 + }, + { + "round": 2, + "arm": "main-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6833e-05, + "prefill_seconds": 2.923957417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.924390292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9254025830000074 + }, + { + "round": 2, + "arm": "main-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6583e-05, + "prefill_seconds": 7.32684075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.327646125, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.329436958000002 + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.7417e-05, + "prefill_seconds": 0.142097667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.14230825, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.1427672080000093 + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4916e-05, + "prefill_seconds": 0.596914375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.597105708, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.5975776660000065 + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4459e-05, + "prefill_seconds": 2.922614041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.923045667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9240610420000053 + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 2.925e-05, + "prefill_seconds": 7.347452875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.348175667, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.349954542000006 + } + ] + } + } +} diff --git a/docs/receipts/lfm2-dspark-review/performance-investigation.json b/docs/receipts/lfm2-dspark-review/performance-investigation.json new file mode 100644 index 0000000000..6bc04e9b0a --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/performance-investigation.json @@ -0,0 +1,1517 @@ +{ + "status": "M4 and M5 comparison complete; no no-loss pass against main", + "published_source": "8bd05874d98d9449ad2f8ddb8796bf26e229874b", + "device": "Apple M4 Pro, 20 GPU cores, 48 GiB", + "protocol": { + "generated_tokens": 256, + "warmup_tokens": 32, + "context": 4096, + "parallel_limit": 16, + "cache": 0, + "sampling": "greedy, seed 1234", + "aggregate_tps": "streams * 255 / (latest last token - earliest first token)", + "arm_order_by_round": [ + [ + "published", + "candidate" + ], + [ + "candidate", + "published" + ], + [ + "published", + "candidate" + ], + [ + "published", + "candidate" + ] + ] + }, + "source_sha256": { + "zig/src/families/lfm2_moe/dspark_block.zig": "d2dd077581904759967446ebe2ad195c083ae73c004862de09907ffb4671ac41", + "zig/src/families/lfm2_moe/dspark.zig": "06b5dda907dcb6f44804edfd0c66466bb7422365516613478c777a6e55f46302", + "zig/src/core/lanes/hold_price.zig": "2e6cbe4c477ae19c9edc527a74471748fc8af012725f402ee4c81ed3bf021dba" + }, + "cells": [ + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 135.60663824374626, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 237.5475197924822, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 270.9110025112889, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 325.10385752712824, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 148.10711121092461, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 270.0303059152453, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 295.6105956217913, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 322.43096510198524, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 43.99498661294338, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 258.03903047977514, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 287.71439774595365, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 329.26775089681786, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 155.17129240785718, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 274.42316268438344, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 293.0455742669573, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 324.3134795211021, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 137.1292625523315, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 252.08277989639137, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.07929135629126, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 327.5277632360025, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 150.82092426693282, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 268.80377082796355, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 294.66891749695225, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 323.9697195614469, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 136.14276531242143, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 252.69384755422968, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.64110269593544, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 325.6922837054757, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 150.98369206524345, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 268.60609628493586, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 293.288959146046, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 323.24421841141884, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 136.28826488314226, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 251.47134877024175, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.5448538421833, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 325.79926741762347, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 150.90184937588137, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 234.90340221222957, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 293.26972222916555, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 322.47025493738624, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 138.34750502292397, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 254.00120377530746, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.5375280962798, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 327.2831823875495, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 151.00505713344813, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 271.8840625705605, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 298.09945887703077, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 324.2089086579436, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 136.32707890068104, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 251.8205867417788, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.349538989573, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 325.9626710889029, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 151.20030809404938, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 268.07908129520496, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 293.3722674786694, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 323.0746966258725, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 138.18860967197506, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 251.84917011393054, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.3401360103994, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 327.6480036147952, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 151.6843393961539, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 268.8005329779927, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 294.4169012094223, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 324.05536974614046, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + } + ], + "limitations": [ + "One candidate lone-code cell was 43.99 tok/s; retained in the four-round median. No competing model process was observed when subsequently checked.", + "Four rounds were not perfectly balanced: the final order repeated published then candidate.", + "Earlier kernel/context candidate regressed M4 16-stream prose by 8.38%; the retained version preserves original context-ingestion arithmetic.", + "An acceptance-cap scheduler experiment lost 2\u20133% on some M5 workloads and is excluded." + ], + "candidate_source": "b4deee277", + "m5": { + "device": "Apple M5 Ultra, 256 GiB; macOS 27.0 (26A428); Zig 0.17.0; Metal 32023.921.6", + "source_sha256": { + "zig/src/core/lanes/windows.zig": "88eedd02202e42e91fcdef4dd046d969038cc391cac282604893a417af3ad13a", + "zig/src/core/lanes/stream.zig": "7a129645b82569a0f9b680088fe35f5e9bd1defb9149e14a8fa85f777ebf48d0", + "zig/src/families/lfm2_moe/dspark_block.zig": "d2dd077581904759967446ebe2ad195c083ae73c004862de09907ffb4671ac41", + "zig/tests/contract/metal_lfm2_bf16.zig": "06f08a81d48a17371bdaed3b1c036aee085af7d7c0e0eed258b845e51a85c774" + }, + "cells": [ + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.99017871497716, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.5857918743773, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 831.2316832063025, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 831.1221850446695, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 261.653824038573, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 622.0862892391041, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 804.3648359431719, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 799.608159390429, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 354.34974736947413, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.481121931619, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 810.5269440440464, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 797.0694737428038, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.3219816253091, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.0402465118209, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 789.0806197623405, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 791.0577258755956, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 354.7364418761545, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 646.0307516882782, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 815.7156281542954, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 798.470003561415, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.5780465065517, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.6789173179606, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 796.0023513285163, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 792.391417523257, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 356.9031926500828, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.8077958032181, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 813.5162265998285, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 796.4582658126303, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.61902076187044, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.9625704407207, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 794.9411091902847, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 790.0706195599455, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 351.7006307459781, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 644.9662400483767, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 813.0724108239679, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 801.6591921437237, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 355.0560737481366, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.0832683633394, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 795.426957848166, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 796.7172061115025, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.0384163078719, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 652.2329226077759, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 831.6711661442548, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 830.8481776811126, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 262.70151064092926, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 622.7723581572978, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 805.1425054893175, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 800.4781222863546, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 353.41984243522757, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.7325520628694, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 810.1592109631267, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 804.1400599123751, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.47342949874684, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.7608129248248, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 789.0114172609233, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 798.1338794206057, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 290.3072664135369, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.6165425845753, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 831.5210047708392, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 829.3651009986223, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 261.76873963295424, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 622.0101018709207, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 804.9815598809657, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 798.5681100696453, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 353.57965911849703, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.0730230346089, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 809.4871362230531, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 797.7172187774961, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.74670092367637, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.772261173814, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 797.1883738155475, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 791.4675338150536, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "binary_sha256": { + "zig-out/native/bin/tensorfold-native": "d5c4358dc174587536ab242aa438fb9fea9557df7a719bef83c35dd9397127c9", + "zig-out/bin/tf-lfm2-dspark": "29519d0208e77721fab62fbf63d982ec8db82a302b603999e185b462b19eb058", + "zig-out/bin/tf-kernel-contract": "4fd08e7265bcec265b92a682f3fe927c7580acf6a1585b69daccfde0936ef6ad" + }, + "protocol": { + "rounds": 3, + "arm_order_by_round": [ + [ + "main", + "published", + "candidate" + ], + [ + "published", + "candidate", + "main" + ], + [ + "candidate", + "main", + "published" + ] + ], + "same_requests_as_m4": true, + "all_72_cells_match": true + }, + "qualification": { + "source": "b4deee277", + "stress": [ + { + "slots": 6, + "seconds": 60, + "completed": 423, + "cancelled": 234, + "wrong": 0, + "errors": [] + }, + { + "slots": 16, + "seconds": 90, + "completed": 827, + "cancelled": 445, + "wrong": 0, + "errors": [] + } + ], + "sampled": [ + { + "streams": 1, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ] + ] + }, + { + "streams": 4, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ] + ] + }, + { + "streams": 8, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ] + ] + }, + { + "streams": 16, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ], + [ + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986" + ], + [ + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351" + ], + [ + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369" + ], + [ + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a" + ], + [ + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363" + ], + [ + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398" + ], + [ + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db" + ], + [ + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd" + ] + ] + } + ], + "resume": { + "equal": true, + "cached_tokens": 3517, + "resumed_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91", + "fresh_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91" + }, + "8bit": [ + { + "streams": 1, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 4, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 8, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 16, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 1, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 4, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 8, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 16, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + } + ], + "long_context": [ + { + "length": 65536, + "plain_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "solo_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "concurrent_sha256": [ + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6" + ] + }, + { + "length": 130816, + "plain_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "solo_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "concurrent_sha256": [ + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3" + ] + } + ], + "kernel_contract": "11 pass, 0 fail; 17 controls red, 0 missed; 0 limit problems", + "drafter_reference": [ + "step 6: confidence reference 0.1449, GPU 0.1465", + "step 7: reference 496 (margin 1.722, second 55658), GPU 496 equal", + "step 7: confidence reference 0.1030, GPU 0.1027", + "step 8: reference 49 (margin 0.100, second 28358), GPU 49 equal", + "step 8: confidence reference 0.0880, GPU 0.0873", + "the reference's largest margin over a different GPU pick: 0.000", + "the confidences' worst difference from the fp32 reference: 0.00158", + "9 of 9 drafts equal the fp32 reference's picks on the same chain (255-token context)" + ] + } + } +} diff --git a/docs/receipts/lfm2-dspark-review/qualification.py b/docs/receipts/lfm2-dspark-review/qualification.py new file mode 100644 index 0000000000..e0987af059 --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/qualification.py @@ -0,0 +1,225 @@ +import concurrent.futures as cf +import fcntl +import json +import os +from pathlib import Path +import random +import signal +import statistics +import subprocess +import sys +import threading +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[3] +OUT = Path(os.environ.get('TF_QUAL_OUT', '/tmp/tf1394-evidence')) +OUT.mkdir(exist_ok=True) +HUB = Path.home()/'.cache/huggingface/hub' +MODEL = Path(os.environ.get('TF_QUAL_MODEL', HUB/'models--LiquidAI--LFM2.5-24B-A2B-MLX-4bit/snapshots/667408d494bde191c2cb9dd4f4549827816b9275')) +DRAFT = Path(os.environ.get('TF_QUAL_DRAFTER', Path.home()/'models/LiquidAI--LFM2.5-24B-A2B-DSpark')) +CAND = Path(os.environ.get('TF_QUAL_CANDIDATE', ROOT/'zig-out/native/bin/tensorfold-native')) +MAIN = Path(os.environ.get('TF_QUAL_MAIN', '/tmp/tf1394-main-base/zig-out/native/bin/tensorfold-native')) +BASE = 'http://127.0.0.1:18194' +PROMPTS = json.loads((ROOT/'tools/bench_prompts.json').read_text())[:8] + +def emit(kind, **data): + print(json.dumps({'event':kind, **data}), flush=True) + +def save(name, data): + (OUT/(name+'.json')).write_text(json.dumps(data, indent=2)+'\n') + +def post(body, stream=False, cancel=None): + body = {'model':'review', 'max_tokens':128, 'temperature':0, 'seed':1234, + 'ignore_eos':True, 'return_token_ids':True, **body, 'stream':stream} + if stream: body['stream_options']={'include_usage':True} + chat='messages' in body + if chat: body['chat_template_kwargs']={'enable_thinking':False} + endpoint='/v1/chat/completions' if chat else '/v1/completions' + req = urllib.request.Request(BASE+endpoint, data=json.dumps(body).encode(), headers={'Content-Type':'application/json'}) + sent = time.perf_counter() + with urllib.request.urlopen(req, timeout=3600) as r: + if not stream: + x=json.load(r) + content=x['choices'][0].get('message',{}).get('content',x['choices'][0].get('text','')) + return {'usage':x['usage'], 'runtime':x['tensorfold'], 'speculative':x.get('speculative',{}), 'content':content, 'wall':time.perf_counter()-sent} + usage={}; runtime={}; spec={}; first=None; last=None; chunks=0; done=False + for raw in r: + line=raw.decode().strip() + if line=='data: [DONE]': done=True; break + if not line.startswith('data:'): continue + x=json.loads(line[5:]) + if 'error' in x: raise RuntimeError(str(x['error'])) + usage=x.get('usage') or usage; runtime=x.get('tensorfold') or runtime; spec=x.get('speculative') or spec + if any(c.get('text') or c.get('delta',{}).get('content') for c in x.get('choices',[])): + now=time.perf_counter(); first=first or now; last=now; chunks+=1 + if cancel and chunks>=cancel: return {'cancelled':True} + if not done or not runtime.get('token_sha256'): raise RuntimeError('incomplete stream') + return {'usage':usage, 'runtime':runtime, 'speculative':spec, 'sent':sent, 'first':first, 'last':last, 'chunks':chunks} + +class Server: + def __init__(self, label, binary=CAND, mode='bf16', context=16384, parallel=16, cache=0, validation=False, model=MODEL, draft=DRAFT): + self.label=label; self.binary=binary; self.mode=mode; self.context=context; self.parallel=parallel + self.cache=cache; self.validation=validation; self.model=model; self.draft=draft + def __enter__(self): + processes=subprocess.check_output(['ps','-Ao','pid=,comm='],text=True) + busy=[line for line in processes.splitlines() if line.split() and Path(line.split()[-1]).name in ('tensorfold-native','tf-lfm2-exact','tf-lfm2-dspark')] + if busy: raise RuntimeError('another model process is already running') + args=[str(self.binary),'serve',str(self.model),'--name','review','--host','127.0.0.1','--port','18194','--parallel',str(self.parallel),'--context',str(self.context),'--prompt-cache-gib',str(self.cache),'--keep-warm','900'] + if self.mode!='plain': args+=['--drafter',str(self.draft)] + if self.mode=='8': args+=['--drafter-bits','8'] + env=dict(os.environ) + if self.validation: env['MTL_DEBUG_LAYER']='1' + self.log=open(OUT/(self.label+'.server.log'),'w') + self.proc=subprocess.Popen(args, stdout=self.log, stderr=subprocess.STDOUT, env=env, start_new_session=True) + start=time.monotonic() + while time.monotonic()-start<600: + if self.proc.poll() is not None: raise RuntimeError(self.label+' server exited') + try: + with urllib.request.urlopen(BASE+'/v1/models',timeout=1) as r: json.load(r) + emit('ready', label=self.label, seconds=time.monotonic()-start) + return self + except Exception: time.sleep(.5) + self.__exit__(None,None,None) + raise TimeoutError('load timeout') + def __exit__(self,*args): + started=time.monotonic(); forced=False + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGTERM) + try: self.proc.wait(timeout=90) + except subprocess.TimeoutExpired: + forced=True; os.killpg(self.proc.pid,signal.SIGKILL); self.proc.wait() + self.log.close() + save(self.label+'.shutdown',{'forced_kill':forced,'seconds':time.monotonic()-started,'returncode':self.proc.returncode}) + if forced: raise RuntimeError(self.label+' required forced shutdown') + +def digest(r): return r['runtime']['token_sha256'] + +def functional(): + results={} + first_messages=[{'role':'user','content':'Public cache fixture.\n'+('alpha beta gamma delta\n'*700)+'\nExplain matrix multiplication in one paragraph.'}] + with Server('functional-cache',cache=1,validation=True): + first=post({'messages':first_messages,'max_tokens':64}) + messages=first_messages+[{'role':'assistant','content':first['content']},{'role':'user','content':'Give a small numerical example and explain each step.'}] + resumed=post({'messages':messages,'max_tokens':64}) + if resumed['usage'].get('prompt_tokens_details',{}).get('cached_tokens',resumed['usage'].get('cached_tokens',0))<=0: + raise AssertionError('resume did not report reused tokens: '+str(resumed['usage'])) + results['resumed']=resumed; results['resume_request']={'messages':messages,'max_tokens':64} + refs=[post({'prompt':p['prompt'],'max_tokens':64}) for p in PROMPTS] + results['stress_reference_hashes']=[{'prompt':p['name'],'sha256':digest(r)} for p,r in zip(PROMPTS,refs)] + results['stress']=[] + for slots,seconds in [(6,60),(16,90)]: + end=time.monotonic()+seconds + def worker(i): + rng=random.Random(4242+i); counts={'completed':0,'cancelled':0,'wrong':0,'errors':[]} + while time.monotonic()0 + sampled=[] + for n in [1,4,8,16]: + bodies=[{'prompt':PROMPTS[i%len(PROMPTS)]['prompt'],'temperature':.7,'top_p':.95,'seed':4242+i,'max_tokens':64} for i in range(n)] + plain=[post({**b,'draft':False}) for b in bodies] + solo=[post(b) for b in bodies] + gate=threading.Barrier(n) + def run(b): gate.wait(); return post(b) + with cf.ThreadPoolExecutor(n) as ex: together=list(ex.map(run,bodies)) + row={'streams':n,'hashes':[[digest(p),digest(s),digest(t)] for p,s,t in zip(plain,solo,together)]} + sampled.append(row); emit('sampled',streams=n,equal=sum(len(set(x))==1 for x in row['hashes'])) + assert all(len(set(x))==1 for x in row['hashes']) + results['sampled']=sampled; save('functional',results) + with Server('functional-fresh'): + fresh=post({'messages':messages,'max_tokens':64}) + assert digest(fresh)==digest(results['resumed']) + results['fresh']=fresh; results['resume_equal']=True; save('functional',results); emit('resume',equal=True) + +def long_context(): + results=[] + with Server('long-context',parallel=1,context=131072): + tail=[825,264,15352,4829,9944,13,271,279,854,22373] + for length in [4096,16384,65536,130816]: + ids=(tail*((length+9)//10))[:length] + plain=post({'prompt':ids,'max_tokens':64,'draft':False}) + drafted=post({'prompt':ids,'max_tokens':64}) + row={'length':length,'plain':plain,'drafted':drafted,'equal':digest(plain)==digest(drafted)} + results.append(row); save('long-context',results); emit('long-context',length=length,equal=row['equal']) + assert row['equal'] + for length in [65536,130816]: + emit('long-reference-start',length=length) + with open(OUT/('dspark-reference-'+str(length)+'.log'),'w') as log: + r=subprocess.run([str(ROOT/'zig-out/bin/tf-lfm2-dspark'),str(MODEL),str(DRAFT),str(length)],stdout=log,stderr=subprocess.STDOUT,timeout=3600) + emit('long-reference-done',length=length,exit=r.returncode) + assert r.returncode==0 + +def performance(family='lfm2', compare=False): + model=MODEL; draft=DRAFT; widths=[1,4,8,16] + if family=='qwen': + model=HUB/'models--TensorFold--Qwen3.8-27B-MLX-4bit/snapshots/22d8d538154e0e5b6f8dcb1fcc74a60a0798104c' + draft=HUB/'models--z-lab--Qwen3.8-27B-DFlash2/snapshots/50307d4c4cde6860d4eee73e2547cd786fe8e8a4' + widths=[1,4] + prompts=[{'name':'code','prompt':'Write a short Python function that computes the Fibonacci sequence and explain it.'},{'name':'chat','prompt':'Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.'}] + arms={'main-plain':(MAIN,'plain'),'candidate-plain':(CAND,'plain'),'candidate-drafted':(CAND,'bf16')} + if family=='qwen': arms={'main-drafted':(MAIN,'bf16'),'candidate-drafted':(CAND,'bf16')} + if compare: + published=os.environ.get('TF_QUAL_PUBLISHED') + if not published: raise SystemExit('compare requires TF_QUAL_PUBLISHED pointing to the earlier PR binary') + arms={'main':(MAIN,'plain'),'published':(Path(published),'bf16'),'candidate':(CAND,'bf16')} + results=[]; reference={} + order=list(arms) + for round_ in range(3): + rotated=order[round_%len(order):]+order[:round_%len(order)] + for arm in rotated: + binary,mode=arms[arm] + with Server(f'perf-{family}-{round_}-{arm}',binary,mode,4096,max(widths),0,False,model,draft): + for p in prompts: post({'prompt':p['prompt'],'max_tokens':32}) + for p in prompts: + for width in widths: + gate=threading.Barrier(width) + def run(i): gate.wait(); return post({'prompt':p['prompt'],'max_tokens':256},True) + with cf.ThreadPoolExecutor(width) as ex: rows=list(ex.map(run,range(width))) + key=p['name']; hashes=[digest(x) for x in rows] + if key not in reference: reference[key]=hashes[0] + assert all(h==reference[key] for h in hashes),(family,arm,key,'token mismatch') + first=min(x['first'] for x in rows); last=max(x['last'] for x in rows) + row={'round':round_,'arm':arm,'prompt':key,'streams':width,'aggregate_decode_tps':width*255/(last-first),'ttft_max':max(x['first']-x['sent'] for x in rows),'requests':rows} + results.append(row); save(('comparison-' if compare else 'performance-')+family,results) + emit('performance',family=family,round=round_,arm=arm,prompt=key,streams=width,tps=round(row['aggregate_decode_tps'],2)) + +def cold_prefill(): + results=[] + arms={'main-plain':(MAIN,'plain'),'candidate-plain':(CAND,'plain'),'candidate-drafted':(CAND,'bf16')} + order=list(arms); cycle=[825,264,15352,4829,9944,13,271,279,854,22373] + for round_ in range(3): + for arm in order[round_:]+order[:round_]: + binary,mode=arms[arm] + with Server(f'cold-{round_}-{arm}',binary,mode,131072,1): + for length in [2048,8192,32768,65536]: + ids=(cycle*((length+9)//10))[:length] + r=post({'prompt':ids,'max_tokens':1}) + assert r['usage']['prompt_tokens']==length + assert r['usage']['prompt_tokens_details']['cached_tokens']==0 + row={'round':round_,'arm':arm,'prompt_tokens':length,**r} + results.append(row); save('cold-prefill',results) + emit('cold-prefill',round=round_,arm=arm,length=length,ttft=r['runtime']['time_to_first_token']) + +if __name__=='__main__': + phase=sys.argv[1] + with open('/tmp/tensorfold-model.lock','a') as lock: + emit('waiting-for-model-lock',phase=phase) + fcntl.flock(lock,fcntl.LOCK_EX) + emit('starting',phase=phase) + if phase=='functional': functional() + elif phase=='long': long_context() + elif phase=='performance': performance(sys.argv[2] if len(sys.argv)>2 else 'lfm2') + elif phase=='compare': performance(compare=True) + elif phase=='cold': cold_prefill() diff --git a/docs/receipts/lfm2-dspark-review/wide-calibration-ablations.json b/docs/receipts/lfm2-dspark-review/wide-calibration-ablations.json new file mode 100644 index 0000000000..020ae3fc71 --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/wide-calibration-ablations.json @@ -0,0 +1,6480 @@ +{ + "status": "diagnostic ablations; not the final qualification", + "base_source": "b4deee277", + "cells": { + "hold-accounting": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 643.8116172492632, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.867035866988, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.5423014255949, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5128631346989, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.2133939448045, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.4367288813849, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 645.8721684421929, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 809.7932585902063, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 799.4562456021381, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 618.8137429002606, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 789.3984943094403, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 793.8443467942008, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 645.5811969949731, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 807.0467578834366, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 795.2596029922896, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 621.5561777345076, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 793.5782789844875, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 792.4435265630628, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.5340653194621, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.5585853396626, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.2073874838301, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 619.1406888714015, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 789.2432337384452, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.516776880887, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.4465736255986, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 816.2421107964317, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 801.4234367658562, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 619.2008415099642, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.1245903763303, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 794.8434530980279, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 644.6861008016517, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 809.9269786391927, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 798.755833324863, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 618.0226926953744, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 789.1412486868718, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 792.3587036469579, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 644.835932459204, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 812.0914984701523, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 799.7322856971059, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 618.7651138042481, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 793.82458947546, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 792.9645002134771, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.7695329258135, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 811.4545693919375, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 800.0092092040525, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.6754308114879, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 790.5291124697678, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.4332636301492, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "overhead-isolation": [ + { + "round": 0, + "arm": "suffix", + "prompt": "code", + "streams": 4, + "tps": 651.9139356848392, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "suffix", + "prompt": "code", + "streams": 8, + "tps": 833.2857206535231, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "suffix", + "prompt": "code", + "streams": 16, + "tps": 835.9149562840696, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "suffix", + "prompt": "prose", + "streams": 4, + "tps": 622.5227827317697, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "suffix", + "prompt": "prose", + "streams": 8, + "tps": 805.6569359731286, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "suffix", + "prompt": "prose", + "streams": 16, + "tps": 804.9583461183761, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "oracle", + "prompt": "code", + "streams": 4, + "tps": 626.6944335227236, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "oracle", + "prompt": "code", + "streams": 8, + "tps": 825.5618638981348, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "oracle", + "prompt": "code", + "streams": 16, + "tps": 807.9078668331922, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "oracle", + "prompt": "prose", + "streams": 4, + "tps": 623.0965393401463, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "oracle", + "prompt": "prose", + "streams": 8, + "tps": 819.1073107777997, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "oracle", + "prompt": "prose", + "streams": 16, + "tps": 801.4843109125941, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "attached-oracle", + "prompt": "code", + "streams": 4, + "tps": 626.0999271344892, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "attached-oracle", + "prompt": "code", + "streams": 8, + "tps": 825.3660475662691, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "attached-oracle", + "prompt": "code", + "streams": 16, + "tps": 808.7132776040836, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 4, + "tps": 622.8070889888528, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 8, + "tps": 819.3872788376917, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 16, + "tps": 802.0719142132929, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "drafted", + "prompt": "code", + "streams": 4, + "tps": 644.4951725611897, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "drafted", + "prompt": "code", + "streams": 8, + "tps": 813.6689741091631, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "drafted", + "prompt": "code", + "streams": 16, + "tps": 798.5743750044154, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "drafted", + "prompt": "prose", + "streams": 4, + "tps": 618.2681572690839, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "drafted", + "prompt": "prose", + "streams": 8, + "tps": 795.5278723676245, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "drafted", + "prompt": "prose", + "streams": 16, + "tps": 791.9102164944356, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "oracle", + "prompt": "code", + "streams": 4, + "tps": 626.7752548814751, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "oracle", + "prompt": "code", + "streams": 8, + "tps": 826.4489176329733, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "oracle", + "prompt": "code", + "streams": 16, + "tps": 814.3368120131368, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "oracle", + "prompt": "prose", + "streams": 4, + "tps": 624.0155122915002, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "oracle", + "prompt": "prose", + "streams": 8, + "tps": 819.9828479123096, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "oracle", + "prompt": "prose", + "streams": 16, + "tps": 807.0855718239884, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "attached-oracle", + "prompt": "code", + "streams": 4, + "tps": 627.5720069660566, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "attached-oracle", + "prompt": "code", + "streams": 8, + "tps": 826.5574111735665, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "attached-oracle", + "prompt": "code", + "streams": 16, + "tps": 807.7942775788831, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 4, + "tps": 623.9850367213689, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 8, + "tps": 820.0032146939689, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 16, + "tps": 799.4948745920863, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "drafted", + "prompt": "code", + "streams": 4, + "tps": 643.34881150891, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "drafted", + "prompt": "code", + "streams": 8, + "tps": 810.2084276073556, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "drafted", + "prompt": "code", + "streams": 16, + "tps": 798.9371330451237, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "drafted", + "prompt": "prose", + "streams": 4, + "tps": 618.4374389899364, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "drafted", + "prompt": "prose", + "streams": 8, + "tps": 797.593366173878, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "drafted", + "prompt": "prose", + "streams": 16, + "tps": 793.8016607458413, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "suffix", + "prompt": "code", + "streams": 4, + "tps": 652.8149539538148, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "suffix", + "prompt": "code", + "streams": 8, + "tps": 832.4678228630254, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "suffix", + "prompt": "code", + "streams": 16, + "tps": 834.935003851258, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "suffix", + "prompt": "prose", + "streams": 4, + "tps": 623.4896993541838, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "suffix", + "prompt": "prose", + "streams": 8, + "tps": 805.5968445877305, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "suffix", + "prompt": "prose", + "streams": 16, + "tps": 806.6520853665469, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "attached-oracle", + "prompt": "code", + "streams": 4, + "tps": 626.7696387571307, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "attached-oracle", + "prompt": "code", + "streams": 8, + "tps": 825.1367948484931, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "attached-oracle", + "prompt": "code", + "streams": 16, + "tps": 811.6072972447571, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 4, + "tps": 622.8303818140437, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 8, + "tps": 818.3383747369702, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "attached-oracle", + "prompt": "prose", + "streams": 16, + "tps": 804.5607618463806, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "drafted", + "prompt": "code", + "streams": 4, + "tps": 644.1967607125431, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "drafted", + "prompt": "code", + "streams": 8, + "tps": 811.2916555717073, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "drafted", + "prompt": "code", + "streams": 16, + "tps": 802.4850089088366, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "drafted", + "prompt": "prose", + "streams": 4, + "tps": 618.8951732623875, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "drafted", + "prompt": "prose", + "streams": 8, + "tps": 790.0171013259228, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "drafted", + "prompt": "prose", + "streams": 16, + "tps": 796.1992414445897, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "suffix", + "prompt": "code", + "streams": 4, + "tps": 652.174121539032, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "suffix", + "prompt": "code", + "streams": 8, + "tps": 831.2944461737667, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "suffix", + "prompt": "code", + "streams": 16, + "tps": 830.5753163876966, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "suffix", + "prompt": "prose", + "streams": 4, + "tps": 621.904087145793, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "suffix", + "prompt": "prose", + "streams": 8, + "tps": 804.5647612762093, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "suffix", + "prompt": "prose", + "streams": 16, + "tps": 799.7848749620825, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "oracle", + "prompt": "code", + "streams": 4, + "tps": 627.0169140117872, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "oracle", + "prompt": "code", + "streams": 8, + "tps": 826.5210064063601, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "oracle", + "prompt": "code", + "streams": 16, + "tps": 807.9543369033614, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "oracle", + "prompt": "prose", + "streams": 4, + "tps": 623.4856976506851, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "oracle", + "prompt": "prose", + "streams": 8, + "tps": 819.3914477642772, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "oracle", + "prompt": "prose", + "streams": 16, + "tps": 800.6927365909038, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "stage512": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.1002045152331, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.9288441624107, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.8704065651071, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.3453986463694, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.8361178213568, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.4605239532948, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 647.0598716246051, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 810.761762946578, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 797.2266217842038, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 619.7187047950405, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 793.6964676341051, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 791.9282262125578, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 645.6156236352848, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 811.4996930937969, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 796.9250218318138, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 619.1732304780877, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.2503544715339, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 790.6492548130739, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 646.3248762331575, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 819.4069301341176, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 800.3348459760032, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 619.1482365327016, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 797.5113084161665, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 794.1212643120977, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.1309082097619, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.3449806490839, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 805.1574545463609, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.9347379131115, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 789.7694274931367, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 798.4068000516954, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 646.753663704301, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 811.5240391620009, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 798.9495509997312, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 620.3189108172958, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.7201955248429, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 792.3067276189817, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 646.4792771524011, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 811.5435304341598, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 798.3027979206439, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 620.4576444123596, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.7101729324681, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 792.2221391903555, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.5922266917154, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 815.1055165738466, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.7626064655209, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5919007482223, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.3130704836641, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.7674920479974, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "phase-profile": [ + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 813.7582181726698, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 803.2588749638213, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.8194277588038, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 796.3293510847767, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "copy-depth": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.5698231050824, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.7629997636277, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 800.0639724877383, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.9960916291277, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 795.9623119914172, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 794.7418540342475, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 635.8934085469378, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 807.7355727142193, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 798.4841326720517, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 619.4876546058689, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.8177306537868, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 792.8294791877873, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 638.8025415387899, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 807.1041519517509, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 800.2587895703317, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 620.0329453289825, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.0620024133443, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 792.5038779934085, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.2277789842284, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 812.2673339855877, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.2146474955022, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.8763193119653, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 791.5135137810487, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.9955237163615, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.5560253073638, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.1310537239268, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.7291201709987, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.7886021016695, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.1646976219921, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.1174413121428, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 638.2320557580033, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 807.8213740625819, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 802.7977304140775, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 620.561499278725, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.2709574286852, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 796.4268285351809, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 638.227196466546, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 807.7983826312196, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 799.0928664875843, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 620.9553872994748, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.0486200617715, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 792.4298028030543, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.2620333699359, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.6097290154547, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.0296424216548, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.5224101622193, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 792.2134225010636, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 796.9240813401055, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "shared-calibration": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 643.8672263414753, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 813.9204466072154, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 802.9999395980669, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.4098645026263, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.4934765134461, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 796.519437923307, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 632.7546042477178, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 812.8668321079678, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 883.5558321243161, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 616.2815390808294, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 794.8857673170639, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 864.9476620371097, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 642.7653023631718, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 813.3024790027683, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 886.3494767826272, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.9106785368859, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.1002468003629, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 863.7629682596074, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.8423531708951, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.731652812001, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.9885752219824, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.1531583535735, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.1099619610673, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.8953852759796, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 646.5601085995847, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 816.3072854901848, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.0876756467267, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 620.0547751768422, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 797.6908152230244, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.9558939987153, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 631.9349732717102, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 818.4119552954095, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 886.1439262176706, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.5384379760573, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 794.5709769513013, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 858.454954141176, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 632.4218011605403, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 809.1906073092416, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 893.9342616519325, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.1052974606797, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.9776983826855, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 863.5671192631019, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.0327904885318, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.5371187032524, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 801.2341625038504, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 619.3005940883706, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.3631061944923, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 794.7691593006111, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "shared-marginals": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 354.5886231883915, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.4363968702983, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 815.142402372856, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.5113247260725, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.2809506501271, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.7709316101826, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.6648510154018, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.1585280677888, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 353.7739736251266, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 634.0287094589685, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 816.4624306041238, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 884.521164233605, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 355.8545247109968, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 616.1133419790036, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.799379381399, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 873.9905629905045, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 352.80406289491833, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 640.5591704758675, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 811.1867558931708, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 884.3521287512664, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 355.4347333932498, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 616.0818501751139, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 794.3131848396016, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 855.1736597157891, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 354.6546246354652, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.2599910212919, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.5617690125395, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.9874073825234, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.3302798758094, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.2849439885823, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.289525016227, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.6706139105225, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 353.80283106837254, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 646.0808791826066, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 817.6687392464844, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 803.463096563945, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.94066175605167, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.753712363033, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.8178065343876, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 796.6235333344594, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 353.1044925522018, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 641.4679975502607, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 817.0524999042318, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 883.7644908150517, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 355.67978776377765, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 616.7057736525313, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.9200860651677, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 866.0757645413831, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 356.73545076881857, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 634.2505232845108, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 808.2288258497729, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 887.379851891824, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 357.8520668127788, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 617.5722305987382, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 797.2344692851619, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 869.1247893304921, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 353.3519734489903, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 646.7524678883676, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 811.3515369455441, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 801.0417076206649, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.6051195970564, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 619.3686599154357, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 790.4226986163708, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 794.2841328092665, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "measured-prices": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 353.0022900836704, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.9552174966029, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.2109343140315, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 800.0504673403519, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 355.97894559035507, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.2240166544733, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.9694197067382, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.6122963222863, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 354.01016860474147, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 640.0832692608378, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 817.2512662810595, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 884.2269914436348, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 355.7104043697476, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.6818234805448, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.4519626733392, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 871.3791667713392, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 355.9226963149118, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 633.7833723015253, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 816.4369707828996, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 891.9506536966762, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 356.46207199869406, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 616.7867743413317, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.6865724814223, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 865.635197618233, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 357.03236762615666, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.7116359452439, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.2915099254717, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 797.9448746694952, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.6425874924987, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.6005297600364, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 789.4981140441996, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.5205934261813, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 356.01922357491236, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.976694323631, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 813.7685110062342, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 801.4986519230725, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 357.2613187916304, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.6761967062737, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 794.976076048426, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 795.080562490925, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 355.67672827826306, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 632.6198973604828, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 814.5936174787505, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 884.0863563323007, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 357.4915866808473, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 616.2536911028309, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.881160887471, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 863.5395353215404, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 355.40053191862216, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 633.304800985909, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 813.0419770200965, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 890.198521413689, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 355.9898165971873, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.3958309968295, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.5456075523746, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 854.7344795212484, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 354.1874092587419, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.0248363588287, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.2082939387749, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.786965293864, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.80092292505157, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.4432510106634, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 795.7560584978334, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.5494222195109, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "marginals-plain-pair": [ + { + "round": 0, + "arm": "suffix", + "prompt": "code", + "streams": 16, + "tps": 852.3819711186778, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "suffix", + "prompt": "prose", + "streams": 16, + "tps": 806.1499360834738, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "drafted", + "prompt": "code", + "streams": 16, + "tps": 883.6145218099846, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "drafted", + "prompt": "prose", + "streams": 16, + "tps": 858.096648332815, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "drafted", + "prompt": "code", + "streams": 16, + "tps": 899.202057482918, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "drafted", + "prompt": "prose", + "streams": 16, + "tps": 853.9158417247984, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "suffix", + "prompt": "code", + "streams": 16, + "tps": 850.6813351383993, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "suffix", + "prompt": "prose", + "streams": 16, + "tps": 804.1592377201139, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "calibrated-copies": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 356.89736524019355, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 646.0187324759729, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 811.741024728307, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 800.8279409915956, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 357.3997695121801, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.9712879245058, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 791.1426154787024, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 795.5312979171122, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 353.52706500963103, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 652.9291232633993, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 834.3728340701181, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 896.0390296891412, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 356.3516920037488, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 607.1620416563453, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 798.5301043013338, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 846.7292730391139, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 354.29895451504433, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 652.0235878379984, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 835.1149319822576, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 893.3051216132275, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 355.54066244519555, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 607.1129835703065, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 797.5689915729046, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 851.143583751972, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 355.0785690913475, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 646.2397188970659, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 813.6934232531094, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.3406355080527, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.1190771418354, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5714398365352, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 795.9064139170582, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.1322137306603, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 354.8496704697536, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.2522864038602, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.7563792827749, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.9494270904845, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.3042033795154, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.7675067526858, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 790.0338392986995, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.2412667992925, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 352.7139871670889, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 653.4023148264475, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 833.2547333810215, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 893.8013643122059, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 355.8086370453564, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 608.1383521142291, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.8971661927403, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 845.9306402571432, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 355.5316157451344, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 653.3602519209168, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 827.3174545834751, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 883.7418866118552, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 354.71168708573737, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 608.7052764114708, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.1492785220543, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 849.1226837895985, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 354.5878835834863, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.8822390501088, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 809.1773804578008, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 800.1935435968933, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 355.4614062043814, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.77057953921, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 788.3009568824, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 797.1140434478928, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "calibrated-gain": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.2817616928971, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.9503439569837, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 803.8429669050786, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5966217081203, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.32050492397, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 797.9728034474932, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 655.3612739129389, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 827.2921520260961, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 898.5541125939604, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 606.792747118611, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.4692566729715, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 821.9535259414042, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 655.2422011190923, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 828.6847884374228, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 894.6143884256593, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 607.2807456583097, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.5083803054556, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 855.4259251192357, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 648.0985030732918, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 816.4987719920528, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.1220168619425, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.3380293389913, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 795.8212232749717, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 791.9775841739299, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.8868087939861, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 812.0902189272058, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 797.0857463577217, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.0180434003239, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 795.494097756085, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 791.2460709694569, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 653.0580187301645, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 835.413492111602, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 895.501733536294, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 607.4044991617268, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.8515252520928, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 832.4465777775944, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 653.4216220485712, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 831.42931875777, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 888.9539789126455, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 608.5829135640643, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 798.6085551885681, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 853.6166076448628, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.6175853573202, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 809.5462435922141, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.0284264896051, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.2663303179743, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 790.1055546163396, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 791.8703317129626, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "matched-copies": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 356.99646309324595, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.5492258005319, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.932885862684, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.4129715234225, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 357.4566124315358, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.514504013522, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 790.6443901963735, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.6781654196443, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 353.9166515876299, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 636.5545013305285, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 812.5945113141222, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 884.1927466504886, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 356.3840436381793, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 616.1584535260791, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.2157635169305, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 893.7677444400998, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 355.37209374435827, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 635.9167330704526, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 817.0502363738213, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 884.0799546586949, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 361.10016599986983, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.1828176051994, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 794.5240802161732, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 865.3193294761426, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 354.909861719153, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.5712374499714, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 814.4099866625494, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 806.3952546542248, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 364.7132316568852, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.8644444174694, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.2592729217582, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 799.9076839089588, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 353.6164532172002, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.5204569248906, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 811.3272957694173, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.8247552364493, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.5363752860445, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 619.1669033437117, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 790.4741917389516, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.224337544272, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 354.3822285167143, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 643.1026126925138, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 817.2482925015051, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 887.860611670729, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 356.2976482608588, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.9026517552672, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.9401589558827, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 863.9163041746773, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 356.5314107668125, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 642.0221936686764, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 818.1466807506825, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 894.9686721601435, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 356.0136938950127, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 614.8136328635114, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.8489442845871, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 869.8564982576298, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 354.3171899590034, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 646.1645619597977, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 815.0506427042201, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.7105753344185, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 357.6725195491522, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5951679719228, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 796.3423294154909, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.7533728687552, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "bounded-confidence": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 353.7680835371117, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 646.4244961929077, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 811.3872230675005, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.6378632970965, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.8454649311719, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 619.0103072043429, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 790.0616695687706, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.1290787796452, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 354.23021507526397, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 639.8778292050813, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 818.2102040596474, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 885.2290261607424, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 309.35306237543205, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.7962295598645, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.8960891596205, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 848.1177087601202, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 355.25533529325133, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 633.9190341655305, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 816.0504998162892, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 888.8641337322866, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 309.14059004467447, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 616.2354478566609, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 794.6971622280608, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 840.2660430574568, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 352.79436176834565, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.7725475329029, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.6250426792632, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.1671109568664, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 355.79316424556134, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.8735213883067, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 790.8595630646305, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.2957843037145, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 353.3220468785289, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.9442405213639, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.3759503855246, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 804.0110891455896, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.20103217979846, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5346952243924, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 789.9944749219242, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 797.1396234203783, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 337.8638929272364, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 633.1106635542818, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 812.8522702261672, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 889.1705482858504, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 309.107753777448, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.7568709687713, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.9285305610407, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 848.7950205273687, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 353.1499910053361, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 632.756452274153, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 811.2639894531362, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 892.7542082734848, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 308.4919982383707, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 615.2449557076353, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.8510800728106, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 841.5034449047317, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 355.47022188523215, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.9453486741286, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 815.271025412628, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.9898000662508, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 357.18306525053754, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 619.3419270640579, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 797.317080241428, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.4595338818068, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "timed-holds": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 353.9551745170593, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.5637468488261, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 807.984763992905, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 801.3765279189241, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.38041169909104, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5369303550488, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 789.8099675056259, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 794.3285718721822, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 318.8819043039091, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 635.7543080729808, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 812.5681185512982, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 866.5437186969713, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 330.05866983125316, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 599.7307974258288, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.8508341728242, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 852.1405741041499, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 309.5573087694549, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 637.0282173004123, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 811.9433552924245, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 864.755605766611, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 319.1617018393784, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 606.9957448592583, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 791.4377684287101, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 843.8802420178199, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 355.5790235851849, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.3876625965851, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 811.5176633383138, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 798.4212262894329, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.35965996407475, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.3677654913529, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 795.1475215096297, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.3545361241212, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 352.9918656139329, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.4623325663915, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 813.3962240076326, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.0035368390922, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.2320914269661, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.9899757409191, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 795.6455306086262, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.9715825415828, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 319.64008095683903, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 632.433123684852, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 807.5030357403751, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 888.4345016760495, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 330.9378911041061, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 613.1681927115468, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 798.5116500646026, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 857.913514905577, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 309.67663598459416, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 628.2013924361069, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 800.2961748637547, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 865.7712022424788, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 330.26751182770295, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 607.0239816131244, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 797.2986689404338, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 844.216850754568, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 355.51257406986366, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.8473415590147, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 813.826905724031, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.0203905043105, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 355.75225545744615, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.0301509608252, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 795.9903673990456, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 3, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 795.2775134771482, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "timed-holds-v2": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 356.69230829755, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.353736357899, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.2646631217915, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 800.3407202667829, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 368.6266218893763, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5935735594454, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 798.2022191584463, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 794.2045453854204, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 312.80102959084525, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 630.5094565366052, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 811.3749866698848, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 883.7831479112893, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 329.9267678631863, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 606.6705659128554, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.2822335082275, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 872.3305412753763, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 309.5502005426481, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 631.7242635070509, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 816.0669308718341, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 889.0877551566958, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 329.0866104337284, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 613.4056960736302, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.3302836453324, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "timed-holds-v3": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 355.86114646948226, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.242574970228, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 809.9454150389603, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 797.3900848713985, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 344.9797439664245, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 617.8759538569072, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 789.1299924459802, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 792.2399449618346, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 306.32830748608166, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 630.5045032779684, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 809.9874900767422, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 892.590961629742, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 317.42741882026826, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 604.5410814558361, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 796.1636361881042, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 852.6768131157207, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 1, + "tps": 319.5727336112319, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 637.1253074031247, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 809.0727058996663, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 885.7462905748039, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 1, + "tps": 318.3155657011862, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 612.6436896536846, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 795.8719992041238, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 841.9255394243237, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 353.3181104124031, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 645.2442586943916, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 810.2098888274344, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 799.6865282121868, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 1, + "tps": 356.3090988418972, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 618.5141289547389, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 788.8790864432178, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 793.3453621679392, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 1, + "tps": 354.0687445295109, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 644.2289378363188, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + } + ] + }, + "incomplete": [ + "timed-holds-v2", + "timed-holds-v3" + ], + "notes": [ + "All runs use public code/prose prompts, greedy seed 1234, 256 tokens, context 4096, cache off.", + "Generic old/live-price labels mean the frozen published drafter baseline and the private experiment.", + "The overhead isolation uses the same frozen binary with suffix, literal draft:false oracle, attached-but-disabled oracle and drafted arms.", + "Phase-profile is instrumented and diagnostic; its rates are not release evidence.", + "Timed-holds-v2/v3 were explicitly interrupted and are not recorded as completed qualification.", + "M4 hold-accounting was interrupted after its controls drifted under unrelated CPU/memory load." + ], + "m4_interrupted_contended_cells": [ + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 255.42985115303972, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 285.8897351673643, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 324.9759499855657, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 230.79741035226908, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 292.03948326150095, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 321.8635097788463, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 250.61604654799328, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 280.35161705555925, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 321.89181286452697, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 233.3064872375181, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 293.4532362980501, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 322.88252957986236, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 254.92513702804413, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 280.49323867369213, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 316.1699880648688, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 228.74850682582047, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 286.0613003174711, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "live-price", + "prompt": "prose", + "streams": 16, + "tps": 295.7224222852996, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 213.9599629051815, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 236.19510069268986, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 268.95459868424456, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 220.88028624348414, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 233.47344218749308, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 262.46936902073395, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 4, + "tps": 199.02623810574727, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 8, + "tps": 178.4980914699512, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "old", + "prompt": "code", + "streams": 16, + "tps": 269.9027915810738, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 4, + "tps": 208.81805728442896, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 8, + "tps": 217.0602358439617, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "old", + "prompt": "prose", + "streams": 16, + "tps": 270.8059638616868, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 4, + "tps": 211.2455862941568, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 8, + "tps": 235.13649439640818, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "code", + "streams": 16, + "tps": 272.7727044611459, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 4, + "tps": 202.21801607515494, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "live-price", + "prompt": "prose", + "streams": 8, + "tps": 230.89681390141425, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + } + ], + "descriptions": { + "hold-accounting": "Models outgoing batched heads from the timed shape rather than per-step multiplication.", + "overhead-isolation": "Same binary: suffix, literal draft:false oracle, drafter attached but disabled, and active drafting.", + "stage512": "128 to 512 staged context rows.", + "phase-profile": "Instrumented host-absorb and GPU-hold phase counters, including warmups.", + "copy-depth": "Conditional acceptance by copied position, learned separately by source.", + "shared-calibration": "Absolute shared-forward timings above 16 rows.", + "shared-marginals": "Shared increments anchored to the original 16-row window cost.", + "measured-prices": "Shared increments plus outgoing hold-shape overhead accounting.", + "marginals-plain-pair": "Same calibrated binary with and without the external drafter.", + "calibrated-copies": "Shared increments with the ordinary fixed copy prior restored.", + "calibrated-gain": "Shared increments, ordinary copy policy, and gain scaled relative to the acceptance prior.", + "matched-copies": "Shared increments with the fixed prior retained for long suffix matches.", + "bounded-confidence": "A deeper pending hold falls back entirely when cached confidence is too short.", + "timed-holds": "Uniform late-hold budgeting; pending work inferred from selected streams.", + "timed-holds-v2": "Uniform budgeting with a drafter-owned pending-cost query; interrupted.", + "timed-holds-v3": "Pending-cost metadata and one proposer query; interrupted." + }, + "phase_profile_totals": { + "includes_warmups": true, + "absorb_calls": 1131, + "absorb_rows": 13102, + "absorb_wall_ms": 17.757, + "explicit_absorb_gpu_ms": 7.58, + "hold_calls": 101, + "hold_gpu_ms": 434.471, + "note": "Hold GPU time also includes staged-context ingestion; these counters are not a pure context-versus-drafting decomposition." + } +} diff --git a/docs/receipts/lfm2-dspark-review/wide-calibration-candidate.json b/docs/receipts/lfm2-dspark-review/wide-calibration-candidate.json new file mode 100644 index 0000000000..2f75052526 --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/wide-calibration-candidate.json @@ -0,0 +1,16059 @@ +{ + "schema_version": 1, + "date": "2026-10-28", + "status": "validated M5 candidate; user accepted the four-stream tradeoff for the larger 16-stream gain; M4 serving qualification is environment-limited", + "source": { + "candidate": "099455490334f03495cdc67de0bfc72174b552b5", + "published": "68c7435b0184a83c7fba458b268bbc5572c0e823", + "main": "dd0a9cc5e503c405a71856631bfeabbccd97169f", + "sha256": { + "zig/src/core/lanes/depth.zig": "06b0c1bfdc1fcb61ad101736e5e7152114ae7f25723b06110910528eafe2728d", + "zig/src/core/lanes/drafted_chances_test.zig": "031300891cbbfff3cf9c792b648815eb9475ccb190158335ace665e66dae6023", + "zig/src/families/lfm2_moe/backend.zig": "16475175e4ec0392ce46e6c993829664f39c4565888de4388953e0192876ecba", + "zig/src/families/lfm2_moe/dspark.zig": "06b5dda907dcb6f44804edfd0c66466bb7422365516613478c777a6e55f46302", + "zig/src/families/lfm2_moe/dspark_block.zig": "d2dd077581904759967446ebe2ad195c083ae73c004862de09907ffb4671ac41", + "zig/src/core/lanes/windows.zig": "88eedd02202e42e91fcdef4dd046d969038cc391cac282604893a417af3ad13a", + "zig/src/core/lanes/stream.zig": "6862f27ff9b86d8f9a6465a5aca8a00403818752e5be947637c806252cff1202", + "zig/tests/contract/metal_lfm2_bf16.zig": "06f08a81d48a17371bdaed3b1c036aee085af7d7c0e0eed258b845e51a85c774" + } + }, + "device": { + "name": "Apple M5 Ultra", + "ram_gib": 256, + "os": "macOS 27.0 (26A428)", + "metal_compiler": "32023.921.6", + "zig": "0.17.0", + "build_options": [ + "-Dcpu=apple_m1", + "-Dversion=1.1.8" + ] + }, + "checkpoint": { + "target": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "revision": "667408d494bde191c2cb9dd4f4549827816b9275", + "drafter": "LiquidAI/LFM2.5-24B-A2B-DSpark", + "drafter_bits": 0, + "sha256": { + "drafter": { + "model.safetensors": "7d06a78556b0b1f386134508f09b00a3a7e0e5e0ec2cc73119229f4ef7c9b299" + }, + "tokenizer.json": "695be7802a0e4b8a81048f0ff5ebb7fc811a0ba5a6be63dbb24deb5a81096f41" + } + }, + "prompts": [ + { + "name": "code", + "prompt": "Write a short Python function that computes the Fibonacci sequence and explain it.", + "count": 14, + "tokens": [ + 22497, + 267, + 2789, + 21516, + 1446, + 395, + 91610, + 278, + 91230, + 8055, + 309, + 6986, + 435, + 22 + ] + }, + { + "name": "prose", + "prompt": "Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.", + "count": 18, + "tokens": [ + 89509, + 1030, + 9706, + 37207, + 5081, + 267, + 54340, + 296, + 15175, + 3478, + 20, + 1256, + 3118, + 267, + 1710, + 20819, + 2122, + 22 + ] + } + ], + "binaries_sha256": { + "main": "3c67809dbda6c2735279fcb0bf084ec21adef406f1f27c6bbbb86bac213e30a5", + "published": "56221194e4dd9f86a030a916fd7971998013089b9023ce72e8cce11c633f81db", + "candidate-plain": "383a32879ec4d4e2c6f8f65bc21de76993c730032a937ce66b996485efa55b7a", + "candidate-drafted": "383a32879ec4d4e2c6f8f65bc21de76993c730032a937ce66b996485efa55b7a" + }, + "protocol": { + "context": 4096, + "parallel_limit": 16, + "prompt_cache_gib": 0, + "max_tokens": 256, + "warmup_tokens": 32, + "temperature": 0, + "seed": 1234, + "ignore_eos": true, + "repetitions": 4, + "aggregate_decode_tps": "streams * 255 / (latest last SSE token - earliest first SSE token)", + "main_and_candidate_plain": "no external drafter; suffix copying remains enabled", + "oracle_controls": "literal draft:false is used separately for correctness", + "arm_order_by_round": [ + [ + "main", + "published", + "candidate-plain", + "candidate-drafted" + ], + [ + "published", + "candidate-plain", + "candidate-drafted", + "main" + ], + [ + "candidate-plain", + "candidate-drafted", + "main", + "published" + ], + [ + "candidate-drafted", + "main", + "published", + "candidate-plain" + ] + ] + }, + "cells": [ + { + "round": 0, + "arm": "main", + "prompt": "code", + "streams": 1, + "tps": 289.3949382948194, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019017125000004853, + "last_s": 0.900165916000006, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 0, + "arm": "main", + "prompt": "code", + "streams": 4, + "tps": 652.4907022857611, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.04169999955684e-05, + "first_s": 0.05019108399999084, + "last_s": 1.576910374999997, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00022637499999689226, + "first_s": 0.033862124999998855, + "last_s": 1.5768938340000034, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00016599999999300508, + "first_s": 0.06655641700000103, + "last_s": 1.5807140000000004, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0, + "first_s": 0.017473334000001728, + "last_s": 1.5768808339999936, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 0, + "arm": "main", + "prompt": "code", + "streams": 8, + "tps": 830.9188714751687, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.087499999793636e-05, + "first_s": 0.03411058300000036, + "last_s": 2.4651412080000057, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0004351249999956508, + "first_s": 0.09942162500000506, + "last_s": 2.4656670410000032, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0005259160000008478, + "first_s": 0.08311704099999417, + "last_s": 2.465258457999994, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00016016600000057224, + "first_s": 0.1320782079999958, + "last_s": 2.4728565409999987, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.00032724999999800275, + "first_s": 0.06678016600000092, + "last_s": 2.4654044159999984, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0002673329999964835, + "first_s": 0.050419457999993256, + "last_s": 2.4652741249999934, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0002102499999949714, + "first_s": 0.017743207999998845, + "last_s": 2.4651232909999976, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0, + "first_s": 0.11576108299999532, + "last_s": 2.465467583000006, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + } + ] + }, + { + "round": 0, + "arm": "main", + "prompt": "code", + "streams": 16, + "tps": 830.6686179686005, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 8.850000000393266e-05, + "first_s": 0.19850612500000864, + "last_s": 4.930106042000006, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0007719590000050403, + "first_s": 0.10046120900000233, + "last_s": 4.929726459000008, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.000659624999997277, + "first_s": 0.11679204200000015, + "last_s": 4.929789749999998, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0005466249999983575, + "first_s": 0.08411750000000495, + "last_s": 4.9300628339999975, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0001528749999977208, + "first_s": 0.13311520899999607, + "last_s": 4.929924999999997, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00024087500000291584, + "first_s": 0.21486808400000257, + "last_s": 4.930217917000007, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0009855420000093318, + "first_s": 0.23116025000000207, + "last_s": 4.929859833999998, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00028745900000615165, + "first_s": 0.03504391699999587, + "last_s": 4.9292779999999965, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0003365419999994401, + "first_s": 0.16581237500000157, + "last_s": 4.930373000000003, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0004827500000033069, + "first_s": 0.14947666700000184, + "last_s": 4.9299721670000025, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0007235839999992777, + "first_s": 0.2638151250000078, + "last_s": 4.930016875000007, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0003879169999976284, + "first_s": 0.06777829199999985, + "last_s": 4.929429917000007, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0009317089999996142, + "first_s": 0.18218070900000782, + "last_s": 4.930336875000009, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0008437919999977339, + "first_s": 0.051403542000002744, + "last_s": 4.929457584000005, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00043537500000923046, + "first_s": 0.01866704200000413, + "last_s": 4.929269458999997, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0, + "first_s": 0.24748458400000573, + "last_s": 4.930299667, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + } + ] + }, + { + "round": 0, + "arm": "main", + "prompt": "prose", + "streams": 1, + "tps": 262.6130764018713, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020124791999990066, + "last_s": 0.9911350839999926, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 0, + "arm": "main", + "prompt": "prose", + "streams": 4, + "tps": 622.8271495637517, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.204199999146567e-05, + "first_s": 0.03935583399999132, + "last_s": 1.6581554589999996, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00023499999998932708, + "first_s": 0.058240541999992956, + "last_s": 1.658177166999991, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00016199999998889325, + "first_s": 0.0770006249999966, + "last_s": 1.6581916250000006, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0, + "first_s": 0.020498083999996197, + "last_s": 1.6581663749999933, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 0, + "arm": "main", + "prompt": "prose", + "streams": 8, + "tps": 804.5637829924607, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.06250000127784e-05, + "first_s": 0.05845654100001241, + "last_s": 2.552360875000005, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003829580000029864, + "first_s": 0.15243112500000677, + "last_s": 2.556268958000004, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00026512500001274475, + "first_s": 0.020733500000005733, + "last_s": 2.552292458000011, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00021220800000776308, + "first_s": 0.09606691600001227, + "last_s": 2.5528507500000046, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00016045800001052157, + "first_s": 0.03959083300000543, + "last_s": 2.5523156250000056, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00046087500000169257, + "first_s": 0.13367604100000108, + "last_s": 2.5526933330000077, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003414160000119182, + "first_s": 0.11484554100000821, + "last_s": 2.5525915830000088, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.0772501659999989, + "last_s": 2.5524381660000017, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 0, + "arm": "main", + "prompt": "prose", + "streams": 16, + "tps": 799.7552905625633, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.183400000267739e-05, + "first_s": 0.2845150840000059, + "last_s": 5.1223552090000055, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005195000000099981, + "first_s": 0.021076792000002342, + "last_s": 5.12148387500001, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00015887499999678312, + "first_s": 0.26571837500000584, + "last_s": 5.122637292000007, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00020879200000933906, + "first_s": 0.15295966700000463, + "last_s": 5.122318042000003, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00040704200000618584, + "first_s": 0.03995979200000477, + "last_s": 5.121496000000008, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00045674999999789634, + "first_s": 0.096516500000007, + "last_s": 5.122189958999996, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007996249999990823, + "first_s": 0.07765324999999734, + "last_s": 5.121597500000007, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003106670000079248, + "first_s": 0.058872542000003136, + "last_s": 5.121506625000009, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00026083400000231904, + "first_s": 0.24694041700000469, + "last_s": 5.122128750000002, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000634458999996923, + "first_s": 0.1341646670000074, + "last_s": 5.121775334000006, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005756249999961938, + "first_s": 0.30331350000000157, + "last_s": 5.122497584000001, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007446250000100463, + "first_s": 0.22816958399999976, + "last_s": 5.122397583999998, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00035950000000184446, + "first_s": 0.11529016699999772, + "last_s": 5.121981042000002, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0008717090000089911, + "first_s": 0.1717496250000039, + "last_s": 5.12244920900001, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006986249999982874, + "first_s": 0.19057083399999897, + "last_s": 5.122595333999996, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.20936879200000647, + "last_s": 5.121805292000005, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 1, + "tps": 352.5059309330252, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.01938350000000355, + "last_s": 0.7427755840000003, + "chunks": 90, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 89, + "drafted": 305, + "accepted": 166, + "acceptance_rate": 0.5442622950819672, + "tokens_per_round": 2.8764044943820224 + } + } + ] + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 4, + "tps": 645.038823928742, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.658300000126019e-05, + "first_s": 0.052342875000007894, + "last_s": 1.5994826670000037, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00023475000000416912, + "first_s": 0.018182500000008872, + "last_s": 1.5955713330000094, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00016962499999806369, + "first_s": 0.035265708000011387, + "last_s": 1.595589583000006, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.0693992080000072, + "last_s": 1.5894960830000002, + "chunks": 223, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 222, + "drafted": 41, + "accepted": 33, + "acceptance_rate": 0.8048780487804879, + "tokens_per_round": 1.1531531531531531 + } + } + ] + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 8, + "tps": 808.7487720963637, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.295899999983703e-05, + "first_s": 0.05481666699999721, + "last_s": 2.5259354169999995, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.00016308399999331868, + "first_s": 0.12275454199999558, + "last_s": 2.525838209, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 29, + "accepted": 22, + "acceptance_rate": 0.7586206896551724, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.00039104199998973854, + "first_s": 0.10571129199999518, + "last_s": 2.525754499999991, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.00021399999999971442, + "first_s": 0.037824791999995, + "last_s": 2.5255334169999912, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0004700839999998152, + "first_s": 0.13968074999999658, + "last_s": 2.540863791999996, + "chunks": 238, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 237, + "drafted": 24, + "accepted": 18, + "acceptance_rate": 0.75, + "tokens_per_round": 1.080168776371308 + } + }, + { + "sent_s": 0.000267624999992222, + "first_s": 0.018448833999997305, + "last_s": 2.525459541999993, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.00033166699999753746, + "first_s": 0.07177470899999605, + "last_s": 2.5255709999999993, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0, + "first_s": 0.0887991249999942, + "last_s": 2.525884167000001, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + } + ] + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 16, + "tps": 798.9815660837332, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.779100000173457e-05, + "first_s": 0.06963183300000253, + "last_s": 5.124508708000008, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0010130410000073198, + "first_s": 0.12072145800000555, + "last_s": 5.124827791000001, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0006217079999970565, + "first_s": 0.18884545800000296, + "last_s": 5.1249649579999925, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0005764579999976149, + "first_s": 0.273867249999995, + "last_s": 5.124401290999998, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00017737500000691853, + "first_s": 0.17184645800000453, + "last_s": 5.124922749999996, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002988330000022188, + "first_s": 0.2058704580000068, + "last_s": 5.125055875000015, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002442079999980251, + "first_s": 0.01855508299999542, + "last_s": 5.123774832999999, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0008294999999947095, + "first_s": 0.08668137499999773, + "last_s": 5.124786916000005, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.000514166000002092, + "first_s": 0.137795416000003, + "last_s": 5.124281749999994, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0006679159999976036, + "first_s": 0.052570290999994995, + "last_s": 5.1237929160000135, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0003599580000042124, + "first_s": 0.23988308299999517, + "last_s": 5.124696707999988, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0007899160000022221, + "first_s": 0.1547861250000011, + "last_s": 5.124875666000008, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0004132079999976668, + "first_s": 0.10369641600000534, + "last_s": 5.1250158750000026, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00046420799999680185, + "first_s": 0.03555520800000522, + "last_s": 5.123854375000008, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.001606541000001016, + "first_s": 0.2568893330000037, + "last_s": 5.124641999999994, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0, + "first_s": 0.22285541600000158, + "last_s": 5.12459233300001, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + } + ] + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 1, + "tps": 356.46165642120616, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.02022737499999039, + "last_s": 0.7355918339999903, + "chunks": 92, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + } + } + ] + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 4, + "tps": 618.1953840111927, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.26660000004631e-05, + "first_s": 0.039795416000004025, + "last_s": 1.6702774999999974, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0002373750000117525, + "first_s": 0.07868741600000817, + "last_s": 1.6703163749999987, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00016404100000499966, + "first_s": 0.020352583000004643, + "last_s": 1.6702853330000096, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0, + "first_s": 0.05923737500000925, + "last_s": 1.6702927500000158, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + } + ] + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 8, + "tps": 788.127553808539, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.304200000315177e-05, + "first_s": 0.15684191700000838, + "last_s": 2.599768291999993, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 10, + "accepted": 4, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0004981250000071213, + "first_s": 0.1374700419999897, + "last_s": 2.5997595839999974, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00033579199998712284, + "first_s": 0.1180892090000043, + "last_s": 2.609401208999998, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 6, + "accepted": 2, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.00026162500000737055, + "first_s": 0.020987708999996357, + "last_s": 2.593058667000008, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 15, + "accepted": 5, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00015808399999173162, + "first_s": 0.09872774999999478, + "last_s": 2.6093892499999924, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 6, + "accepted": 2, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.00038604200000236233, + "first_s": 0.07931337499999813, + "last_s": 2.5932052919999933, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 8, + "accepted": 5, + "acceptance_rate": 0.625, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00020895899999118228, + "first_s": 0.059855709000004254, + "last_s": 2.593099125000009, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 8, + "accepted": 5, + "acceptance_rate": 0.625, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0, + "first_s": 0.04044404199999008, + "last_s": 2.5930863339999917, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 9, + "accepted": 5, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.024 + } + } + ] + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 16, + "tps": 792.5756383104019, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.562500000015461e-05, + "first_s": 0.15681554200000392, + "last_s": 5.168663334000001, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0007145419999972091, + "first_s": 0.2733495420000054, + "last_s": 5.168337708999985, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0003699589999826003, + "first_s": 0.23450595899998916, + "last_s": 5.168161709000003, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00021504200000777018, + "first_s": 0.07925975000000562, + "last_s": 5.167919041999994, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00016416700000831952, + "first_s": 0.13745066699999597, + "last_s": 5.168629041999992, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00041904200000431047, + "first_s": 0.11803787499999885, + "last_s": 5.168748625000006, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0005311669999912283, + "first_s": 0.05986920899999859, + "last_s": 5.1685104999999965, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0010202499999820702, + "first_s": 0.19564241699998774, + "last_s": 5.168433416999989, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0002687089999824366, + "first_s": 0.02104024999999865, + "last_s": 5.167743124999987, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0006473749999997835, + "first_s": 0.21508354200000213, + "last_s": 5.168813916999994, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0003192089999970449, + "first_s": 0.04043733399998928, + "last_s": 5.167883125000003, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.000785250000006954, + "first_s": 0.2927850000000092, + "last_s": 5.168283833999993, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.000600833999982342, + "first_s": 0.2539376670000024, + "last_s": 5.168594999999982, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0009392919999982041, + "first_s": 0.31220487499999194, + "last_s": 5.168086458999994, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0004664999999874908, + "first_s": 0.09865124999998898, + "last_s": 5.168122667000006, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0, + "first_s": 0.17620324999998616, + "last_s": 5.168553541999984, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "tps": 289.664898526758, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.01763933300000531, + "last_s": 0.8979669160000014, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "tps": 661.8283265634324, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00011133300000665258, + "first_s": 0.03458058300000744, + "last_s": 1.5593261660000053, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 63, + "accepted": 37, + "acceptance_rate": 0.5873015873015873, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.0002732499999922311, + "first_s": 0.0673262500000078, + "last_s": 1.5593379999999968, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.00019895799999858355, + "first_s": 0.050959957999992866, + "last_s": 1.5593013330000076, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.0, + "first_s": 0.018152832999987822, + "last_s": 1.5593156660000034, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 63, + "accepted": 37, + "acceptance_rate": 0.5873015873015873, + "tokens_per_round": 1.1743119266055047 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "tps": 852.2134767889547, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.850000000710679e-05, + "first_s": 0.10013691599999675, + "last_s": 2.4077089580000006, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0005052500000033433, + "first_s": 0.13276054100001033, + "last_s": 2.4121705409999947, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0004542910000111533, + "first_s": 0.116439583000016, + "last_s": 2.4121618750000096, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0003588749999892116, + "first_s": 0.05117420799999195, + "last_s": 2.4075726250000002, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0001736250000021755, + "first_s": 0.018404124999989335, + "last_s": 2.4072412080000163, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0002284159999987878, + "first_s": 0.06752266600000212, + "last_s": 2.407417874999993, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0002847080000094593, + "first_s": 0.03478341600001045, + "last_s": 2.407264583, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.08383995799999866, + "last_s": 2.407447458000007, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "tps": 849.1748999161405, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00010283399998911591, + "first_s": 0.11666416700001037, + "last_s": 4.814435916999997, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.000866624999986243, + "first_s": 0.16564487499999814, + "last_s": 4.814162417000006, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00017866700000013225, + "first_s": 0.08403904200000056, + "last_s": 4.813875999999993, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00023091700001032223, + "first_s": 0.0677802090000057, + "last_s": 4.813584166999988, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00034616700000356104, + "first_s": 0.10040054199998849, + "last_s": 4.814117374999995, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0008200419999866426, + "first_s": 0.19820616699999505, + "last_s": 4.814219667000003, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0005775420000020404, + "first_s": 0.13295387499999833, + "last_s": 4.814350917000013, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00028962499999352076, + "first_s": 0.0350679590000027, + "last_s": 4.813511459000011, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0006246670000109589, + "first_s": 0.21453454199999555, + "last_s": 4.813862416999996, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0004491669999993064, + "first_s": 0.26347254199998815, + "last_s": 4.8232971250000105, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 49, + "accepted": 19, + "acceptance_rate": 0.3877551020408163, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.0007704589999946165, + "first_s": 0.05141445900000008, + "last_s": 4.813496708999992, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0006760420000091472, + "first_s": 0.23084758400000283, + "last_s": 4.814389709000011, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.000498625000005859, + "first_s": 0.1819189999999935, + "last_s": 4.81455895900001, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0003985840000098051, + "first_s": 0.1493125419999899, + "last_s": 4.814299833999996, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0007239169999877504, + "first_s": 0.24718016700001044, + "last_s": 4.823283500000002, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 49, + "accepted": 19, + "acceptance_rate": 0.3877551020408163, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.0, + "first_s": 0.01863320900000076, + "last_s": 4.813484875, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 1, + "tps": 261.38390992402503, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.02007799999998383, + "last_s": 0.9956545000000006, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 4, + "tps": 609.9669343383413, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.233299999777955e-05, + "first_s": 0.07724037500000236, + "last_s": 1.6928297499999871, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.00025262500000167165, + "first_s": 0.03951279099999283, + "last_s": 1.692698665999984, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 35, + "accepted": 8, + "acceptance_rate": 0.22857142857142856, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.00016170799997894392, + "first_s": 0.02060795799999937, + "last_s": 1.6926645000000065, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 35, + "accepted": 8, + "acceptance_rate": 0.22857142857142856, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.0, + "first_s": 0.058399415999986104, + "last_s": 1.692706749999985, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 8, + "tps": 809.672115751542, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.00010250000002542947, + "first_s": 0.05841479100001834, + "last_s": 2.539832791000009, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0004066660000034972, + "first_s": 0.13364954100001114, + "last_s": 2.540146375000006, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00018041600000628932, + "first_s": 0.03954866600000173, + "last_s": 2.5398536660000275, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00046229100001937695, + "first_s": 0.09605575000000499, + "last_s": 2.5398657500000184, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0005358330000149181, + "first_s": 0.11484220800002731, + "last_s": 2.5402444160000073, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0002577500000029431, + "first_s": 0.07728983300000891, + "last_s": 2.5399477079999997, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00034770800002092983, + "first_s": 0.1524370410000131, + "last_s": 2.5402004160000047, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0, + "first_s": 0.02070600000001832, + "last_s": 2.539821083000021, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 16, + "tps": 801.9628172039064, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.045800001672433e-05, + "first_s": 0.021071375000019543, + "last_s": 5.103436583000018, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007646250000163946, + "first_s": 0.2842919160000008, + "last_s": 5.108578083000026, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 35, + "accepted": 3, + "acceptance_rate": 0.08571428571428572, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002085410000063348, + "first_s": 0.1715162080000141, + "last_s": 5.103870625000013, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005340000000160217, + "first_s": 0.11514520800000128, + "last_s": 5.104300083000027, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00015829100001951701, + "first_s": 0.20912250000000654, + "last_s": 5.1037983330000145, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00025845800001889074, + "first_s": 0.05869633300000032, + "last_s": 5.103474958000021, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00040904100001171173, + "first_s": 0.24668070800001374, + "last_s": 5.104044791000007, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006332910000139691, + "first_s": 0.09634137500000861, + "last_s": 5.104135041000006, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005792910000081974, + "first_s": 0.1527348330000109, + "last_s": 5.104255750000021, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00035791600001289225, + "first_s": 0.2655055830000208, + "last_s": 5.10409525, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000925416000001178, + "first_s": 0.22789587500000152, + "last_s": 5.104215083000014, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003089160000229185, + "first_s": 0.03990633300000468, + "last_s": 5.103451416000013, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007230410000147458, + "first_s": 0.13391554100002168, + "last_s": 5.103779375000016, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006800410000096235, + "first_s": 0.3030480830000215, + "last_s": 5.108589041000016, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 35, + "accepted": 3, + "acceptance_rate": 0.08571428571428572, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00045841600001494953, + "first_s": 0.07749095800002692, + "last_s": 5.103549541000007, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.19033883300002685, + "last_s": 5.104173500000002, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "tps": 354.03697603829806, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019182792000009385, + "last_s": 0.739446541999996, + "chunks": 90, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 89, + "drafted": 305, + "accepted": 166, + "acceptance_rate": 0.5442622950819672, + "tokens_per_round": 2.8764044943820224 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "tps": 632.6567488487849, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00010625000001596163, + "first_s": 0.06942499999999541, + "last_s": 1.630453541999998, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 54, + "accepted": 33, + "acceptance_rate": 0.6111111111111112, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0002620000000206346, + "first_s": 0.03533179199999381, + "last_s": 1.605780167000006, + "chunks": 221, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 220, + "drafted": 53, + "accepted": 35, + "acceptance_rate": 0.660377358490566, + "tokens_per_round": 1.1636363636363636 + } + }, + { + "sent_s": 0.00018633300001624775, + "first_s": 0.05237804200001506, + "last_s": 1.6304402080000102, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 55, + "accepted": 33, + "acceptance_rate": 0.6, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.018204875000009224, + "last_s": 1.6057571249999967, + "chunks": 221, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 220, + "drafted": 54, + "accepted": 35, + "acceptance_rate": 0.6481481481481481, + "tokens_per_round": 1.1636363636363636 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "tps": 816.3843674754631, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.166699999241246e-05, + "first_s": 0.08664674999999988, + "last_s": 2.4797165829999983, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 41, + "accepted": 30, + "acceptance_rate": 0.7317073170731707, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.000474624999981188, + "first_s": 0.12073754200000053, + "last_s": 2.461433707999987, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 46, + "accepted": 32, + "acceptance_rate": 0.6956521739130435, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0001684169999975893, + "first_s": 0.018413541999990457, + "last_s": 2.4714994579999825, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 42, + "accepted": 31, + "acceptance_rate": 0.7380952380952381, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00022629199997936666, + "first_s": 0.06960504199997786, + "last_s": 2.4797044169999936, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 44, + "accepted": 30, + "acceptance_rate": 0.6818181818181818, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0003487500000005639, + "first_s": 0.05257195799998726, + "last_s": 2.4715415829999756, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 41, + "accepted": 31, + "acceptance_rate": 0.7560975609756098, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0004019999999798074, + "first_s": 0.10369166699999255, + "last_s": 2.479870874999989, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 39, + "accepted": 30, + "acceptance_rate": 0.7692307692307693, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0002774999999815009, + "first_s": 0.1377821250000011, + "last_s": 2.5172364999999957, + "chunks": 230, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 229, + "drafted": 42, + "accepted": 26, + "acceptance_rate": 0.6190476190476191, + "tokens_per_round": 1.1179039301310043 + } + }, + { + "sent_s": 0.0, + "first_s": 0.035519082999996954, + "last_s": 2.4715244169999835, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 41, + "accepted": 31, + "acceptance_rate": 0.7560975609756098, + "tokens_per_round": 1.1428571428571428 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "tps": 886.0534135309714, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.408400001120754e-05, + "first_s": 0.03568766700001902, + "last_s": 4.383103042000016, + "chunks": 209, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 208, + "drafted": 68, + "accepted": 47, + "acceptance_rate": 0.6911764705882353, + "tokens_per_round": 1.2307692307692308 + } + }, + { + "sent_s": 0.0008179170000062186, + "first_s": 0.18918575000000715, + "last_s": 4.623324875000009, + "chunks": 229, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 228, + "drafted": 39, + "accepted": 27, + "acceptance_rate": 0.6923076923076923, + "tokens_per_round": 1.1228070175438596 + } + }, + { + "sent_s": 0.00016258400000879192, + "first_s": 0.08681654200000821, + "last_s": 4.619348416999998, + "chunks": 228, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 227, + "drafted": 46, + "accepted": 28, + "acceptance_rate": 0.6086956521739131, + "tokens_per_round": 1.1277533039647578 + } + }, + { + "sent_s": 0.0005443750000040382, + "first_s": 0.22330387499999915, + "last_s": 4.593885999999998, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 45, + "accepted": 32, + "acceptance_rate": 0.7111111111111111, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0002161670000191407, + "first_s": 0.2574249170000087, + "last_s": 4.025517334, + "chunks": 191, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 190, + "drafted": 93, + "accepted": 65, + "acceptance_rate": 0.6989247311827957, + "tokens_per_round": 1.3473684210526315 + } + }, + { + "sent_s": 0.0002708339999912823, + "first_s": 0.20626833400001487, + "last_s": 4.607208041999996, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 39, + "accepted": 30, + "acceptance_rate": 0.7692307692307693, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0003712920000111808, + "first_s": 0.0697367920000147, + "last_s": 4.578519083999993, + "chunks": 222, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 221, + "drafted": 50, + "accepted": 34, + "acceptance_rate": 0.68, + "tokens_per_round": 1.158371040723982 + } + }, + { + "sent_s": 0.0009464999999977408, + "first_s": 0.137989417, + "last_s": 4.4582448339999985, + "chunks": 214, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 213, + "drafted": 50, + "accepted": 42, + "acceptance_rate": 0.84, + "tokens_per_round": 1.2018779342723005 + } + }, + { + "sent_s": 0.0005899170000134291, + "first_s": 0.27448724999999286, + "last_s": 4.442635249999995, + "chunks": 213, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 212, + "drafted": 62, + "accepted": 43, + "acceptance_rate": 0.6935483870967742, + "tokens_per_round": 1.2075471698113207 + } + }, + { + "sent_s": 0.00042354199999294906, + "first_s": 0.2403498749999926, + "last_s": 4.619380709000012, + "chunks": 228, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 227, + "drafted": 46, + "accepted": 28, + "acceptance_rate": 0.6086956521739131, + "tokens_per_round": 1.1277533039647578 + } + }, + { + "sent_s": 0.0003216670000085742, + "first_s": 0.052726875000018936, + "last_s": 4.49220895900001, + "chunks": 216, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 215, + "drafted": 60, + "accepted": 40, + "acceptance_rate": 0.6666666666666666, + "tokens_per_round": 1.1906976744186046 + } + }, + { + "sent_s": 0.0006358340000076623, + "first_s": 0.12092820899999879, + "last_s": 4.244078042000012, + "chunks": 202, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 201, + "drafted": 66, + "accepted": 54, + "acceptance_rate": 0.8181818181818182, + "tokens_per_round": 1.2736318407960199 + } + }, + { + "sent_s": 0.0007048340000039843, + "first_s": 0.10387600000001385, + "last_s": 4.427200499999998, + "chunks": 212, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 211, + "drafted": 62, + "accepted": 44, + "acceptance_rate": 0.7096774193548387, + "tokens_per_round": 1.2132701421800949 + } + }, + { + "sent_s": 0.0007632920000162358, + "first_s": 0.15505937500000755, + "last_s": 4.367775167000019, + "chunks": 208, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 207, + "drafted": 60, + "accepted": 48, + "acceptance_rate": 0.8, + "tokens_per_round": 1.2367149758454106 + } + }, + { + "sent_s": 0.0004777919999980895, + "first_s": 0.018636334000007082, + "last_s": 4.222254334000013, + "chunks": 201, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 200, + "drafted": 87, + "accepted": 55, + "acceptance_rate": 0.632183908045977, + "tokens_per_round": 1.28 + } + }, + { + "sent_s": 0.0, + "first_s": 0.17210966700000085, + "last_s": 4.458342250000015, + "chunks": 214, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 213, + "drafted": 51, + "accepted": 42, + "acceptance_rate": 0.8235294117647058, + "tokens_per_round": 1.2018779342723005 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 1, + "tps": 357.0286814706803, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020285457999989376, + "last_s": 0.7345137910000119, + "chunks": 92, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 4, + "tps": 615.2343022307355, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.7167000006948e-05, + "first_s": 0.020550209000020914, + "last_s": 1.678386209000024, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00024150000001554872, + "first_s": 0.0787438750000149, + "last_s": 1.6784551670000099, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00017075000002364504, + "first_s": 0.05935804200001371, + "last_s": 1.678423500000008, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0, + "first_s": 0.040020125000012285, + "last_s": 1.6783760000000143, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 16, + "accepted": 6, + "acceptance_rate": 0.375, + "tokens_per_round": 1.0281124497991967 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 8, + "tps": 794.7774420026692, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.837499999321153e-05, + "first_s": 0.04286704099999383, + "last_s": 2.573487790999991, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00035412500000120417, + "first_s": 0.14019941600000152, + "last_s": 2.5737503329999925, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 16, + "accepted": 5, + "acceptance_rate": 0.3125, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00017374999998764906, + "first_s": 0.021025666000014098, + "last_s": 2.5734014580000064, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.000523958000002267, + "first_s": 0.15965133299999934, + "last_s": 2.5739015830000085, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 15, + "accepted": 5, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00022058299998661823, + "first_s": 0.06268308299999603, + "last_s": 2.573841625, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0004306659999997464, + "first_s": 0.08208716600000798, + "last_s": 2.5877701249999916, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 5, + "accepted": 2, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.00028129099999318896, + "first_s": 0.10145412500000361, + "last_s": 2.5877819579999937, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 5, + "accepted": 2, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.0, + "first_s": 0.12079104099998972, + "last_s": 2.573796291000008, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 9, + "accepted": 5, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.024 + } + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 16, + "tps": 868.6711590769241, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 8.066699999176308e-05, + "first_s": 0.1375496669999734, + "last_s": 4.6933760829999756, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 19, + "accepted": 8, + "acceptance_rate": 0.42105263157894735, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.00014508299997828544, + "first_s": 0.11816745799998785, + "last_s": 4.600768082999991, + "chunks": 240, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 30, + "accepted": 16, + "acceptance_rate": 0.5333333333333333, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00090862499999389, + "first_s": 0.3120279579999874, + "last_s": 4.717991749999982, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 16, + "accepted": 5, + "acceptance_rate": 0.3125, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0005382919999874503, + "first_s": 0.21509041699999898, + "last_s": 4.717809416999984, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00020049999997695522, + "first_s": 0.25389033299998687, + "last_s": 4.693535666999992, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 14, + "accepted": 8, + "acceptance_rate": 0.5714285714285714, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.0002541249999978845, + "first_s": 0.02116262499998811, + "last_s": 4.710796749999986, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 22, + "accepted": 6, + "acceptance_rate": 0.2727272727272727, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00035341699998525655, + "first_s": 0.17631412499997623, + "last_s": 4.71082741699999, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 20, + "accepted": 6, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0007031669999832957, + "first_s": 0.07942274999999199, + "last_s": 4.11868149999998, + "chunks": 206, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 205, + "drafted": 75, + "accepted": 50, + "acceptance_rate": 0.6666666666666666, + "tokens_per_round": 1.248780487804878 + } + }, + { + "sent_s": 0.0007799169999884725, + "first_s": 0.2344351249999761, + "last_s": 4.717902916999975, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 11, + "accepted": 5, + "acceptance_rate": 0.45454545454545453, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0005970829999739635, + "first_s": 0.195703457999997, + "last_s": 4.717783082999972, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00030537499998217754, + "first_s": 0.27325720799998976, + "last_s": 4.310620916999994, + "chunks": 219, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 45, + "accepted": 37, + "acceptance_rate": 0.8222222222222222, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.0004062919999796577, + "first_s": 0.09882720799998879, + "last_s": 4.454720457999997, + "chunks": 229, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 228, + "drafted": 43, + "accepted": 27, + "acceptance_rate": 0.627906976744186, + "tokens_per_round": 1.1228070175438596 + } + }, + { + "sent_s": 0.0008489999999881093, + "first_s": 0.29266841699998736, + "last_s": 3.9745678329999805, + "chunks": 197, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 196, + "drafted": 76, + "accepted": 59, + "acceptance_rate": 0.7763157894736842, + "tokens_per_round": 1.3061224489795917 + } + }, + { + "sent_s": 0.0006461249999745178, + "first_s": 0.040601624999993646, + "last_s": 4.600757082999991, + "chunks": 240, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 34, + "accepted": 16, + "acceptance_rate": 0.47058823529411764, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00045437499997547093, + "first_s": 0.15691949999998656, + "last_s": 4.682362041999994, + "chunks": 247, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 246, + "drafted": 26, + "accepted": 9, + "acceptance_rate": 0.34615384615384615, + "tokens_per_round": 1.0406504065040652 + } + }, + { + "sent_s": 0.0, + "first_s": 0.060019041999993306, + "last_s": 4.702711332999996, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 15, + "accepted": 7, + "acceptance_rate": 0.4666666666666667, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 1, + "tps": 353.875518041669, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019534999999990532, + "last_s": 0.7401273749999859, + "chunks": 90, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 89, + "drafted": 305, + "accepted": 166, + "acceptance_rate": 0.5442622950819672, + "tokens_per_round": 2.8764044943820224 + } + } + ] + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 4, + "tps": 646.3712093265011, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00010608300001990756, + "first_s": 0.052181250000018053, + "last_s": 1.596157040999998, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00024112500000228465, + "first_s": 0.018116458000008606, + "last_s": 1.5922712079999997, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00018091600000502694, + "first_s": 0.03516187500000001, + "last_s": 1.5922490410000023, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.06923041600001056, + "last_s": 1.5861764160000007, + "chunks": 223, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 222, + "drafted": 40, + "accepted": 33, + "acceptance_rate": 0.825, + "tokens_per_round": 1.1531531531531531 + } + } + ] + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 8, + "tps": 815.6355603289662, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0001041659999998501, + "first_s": 0.01831516600000782, + "last_s": 2.5028167080000117, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0004861250000089967, + "first_s": 0.0865736659999925, + "last_s": 2.503194707999995, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0003014579999955913, + "first_s": 0.052476250000012215, + "last_s": 2.502932874999999, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0001798750000148175, + "first_s": 0.035417916000000105, + "last_s": 2.502847666000008, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0002463749999890297, + "first_s": 0.10356395799999518, + "last_s": 2.503322166000004, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0003758329999925536, + "first_s": 0.0695484160000035, + "last_s": 2.5032495410000024, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.000426790999995319, + "first_s": 0.12060600000000932, + "last_s": 2.507813999999996, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 28, + "accepted": 21, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0, + "first_s": 0.13762124999999514, + "last_s": 2.519432208000012, + "chunks": 238, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 237, + "drafted": 24, + "accepted": 18, + "acceptance_rate": 0.75, + "tokens_per_round": 1.080168776371308 + } + } + ] + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 16, + "tps": 798.7719859603106, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.37920000012582e-05, + "first_s": 0.0356257919999905, + "last_s": 5.125112999999999, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0009933340000145563, + "first_s": 0.273747458999992, + "last_s": 5.126361125000017, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0009274170000139748, + "first_s": 0.17177483400001847, + "last_s": 5.125810833999992, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00016720900001132577, + "first_s": 0.13777945900000077, + "last_s": 5.126105499999994, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0005507920000127342, + "first_s": 0.06964479199999118, + "last_s": 5.125645875000004, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00022087499999656757, + "first_s": 0.12081962499999577, + "last_s": 5.126271792000011, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0005977090000044427, + "first_s": 0.08671187499999178, + "last_s": 5.126017750000017, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002775419999920814, + "first_s": 0.20584670900001356, + "last_s": 5.126156417000004, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00048458399999162793, + "first_s": 0.052683625000014445, + "last_s": 5.1251904590000095, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0003363750000175969, + "first_s": 0.1547320840000168, + "last_s": 5.125751459000014, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0008224169999948572, + "first_s": 0.23987316700001315, + "last_s": 5.125893084000012, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00038716700001373283, + "first_s": 0.018572792000014715, + "last_s": 5.125086459000016, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0006584589999931723, + "first_s": 0.10379062500001623, + "last_s": 5.125585125000015, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00043458400000417896, + "first_s": 0.25679174999999077, + "last_s": 5.126311250000015, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0007168749999948432, + "first_s": 0.22288866700000654, + "last_s": 5.125942750000007, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0, + "first_s": 0.18882725000000278, + "last_s": 5.126413417000009, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + } + ] + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 1, + "tps": 357.08738506794214, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.02016437500000734, + "last_s": 0.7342752920000066, + "chunks": 92, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + } + } + ] + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 4, + "tps": 619.427853620994, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0001016669999955866, + "first_s": 0.020417124999994485, + "last_s": 1.667055375000018, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0002669590000152766, + "first_s": 0.03986750000001393, + "last_s": 1.66709800000001, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0001857919999963542, + "first_s": 0.0787461670000198, + "last_s": 1.6670894169999997, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0, + "first_s": 0.05925091700001417, + "last_s": 1.6670743749999986, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + } + ] + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 8, + "tps": 796.5509863951829, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 8.554199999366574e-05, + "first_s": 0.07897958300000596, + "last_s": 2.5782102080000016, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0004218750000006821, + "first_s": 0.1565373749999992, + "last_s": 2.581858208, + "chunks": 256, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 255, + "drafted": 6, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + } + }, + { + "sent_s": 0.00024970799998413895, + "first_s": 0.05960837499998206, + "last_s": 2.5779354999999953, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00019904199999132288, + "first_s": 0.09835508300000129, + "last_s": 2.5783844999999985, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00015183299998966504, + "first_s": 0.04022995799999762, + "last_s": 2.5778847499999813, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00037075000000186265, + "first_s": 0.11774279200000137, + "last_s": 2.578158583000004, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00032183299998678194, + "first_s": 0.13712483300000144, + "last_s": 2.578087457999999, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 3, + "accepted": 1, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0, + "first_s": 0.020816874999979973, + "last_s": 2.577838208000003, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + } + ] + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 16, + "tps": 793.0442952546944, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.337500000583532e-05, + "first_s": 0.254272708000002, + "last_s": 5.1658878749999815, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0005447079999783, + "first_s": 0.0990860830000031, + "last_s": 5.165940250000006, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00016712499999016472, + "first_s": 0.060306166000003714, + "last_s": 5.165142000000003, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0005903329999910056, + "first_s": 0.040925082999990536, + "last_s": 5.165075833000003, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0006879999999966913, + "first_s": 0.1572639160000051, + "last_s": 5.166023082999999, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00023554099999500977, + "first_s": 0.07968662499999368, + "last_s": 5.165524790999996, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00028979100000015023, + "first_s": 0.021523665999978903, + "last_s": 5.165043374999982, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0007354999999904521, + "first_s": 0.118454665999991, + "last_s": 5.165688208000006, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00044608299998571965, + "first_s": 0.292976082999985, + "last_s": 5.165848041000004, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0006405000000029304, + "first_s": 0.17663833299999965, + "last_s": 5.165749874999989, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0003424999999879219, + "first_s": 0.3124002909999888, + "last_s": 5.166113290999988, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0007830829999875277, + "first_s": 0.23488570799997888, + "last_s": 5.165363208000002, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0008322079999913967, + "first_s": 0.273624749999982, + "last_s": 5.1662552079999955, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00039424999999937427, + "first_s": 0.19606749999999806, + "last_s": 5.166167624999986, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0004940000000033251, + "first_s": 0.13784566600000403, + "last_s": 5.165808124999984, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0, + "first_s": 0.21552279099998373, + "last_s": 5.1659797079999805, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "tps": 288.3251495763636, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.01751966700001617, + "last_s": 0.9019378330000052, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "tps": 661.7471569888854, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.295800001041243e-05, + "first_s": 0.017952000000008184, + "last_s": 1.559315708000014, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 63, + "accepted": 37, + "acceptance_rate": 0.5873015873015873, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.00023954200000275705, + "first_s": 0.06702970800000685, + "last_s": 1.559305749999993, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.00016483300001368661, + "first_s": 0.05068258299999684, + "last_s": 1.5593262080000159, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.0, + "first_s": 0.03432762500000308, + "last_s": 1.559288874999993, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "tps": 852.361559328986, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.658299998704933e-05, + "first_s": 0.13285808300000213, + "last_s": 2.4116938330000153, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0004410409999877629, + "first_s": 0.08385429099999442, + "last_s": 2.4072955409999963, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00017108299999790688, + "first_s": 0.10018154100001198, + "last_s": 2.407141375000009, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00034004100001538973, + "first_s": 0.051151207999993176, + "last_s": 2.4069557500000087, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00027695800000060444, + "first_s": 0.018343291000007866, + "last_s": 2.406800583000006, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00022220799999672636, + "first_s": 0.06750845799999183, + "last_s": 2.4070947909999916, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00038791599999399295, + "first_s": 0.11651170799999022, + "last_s": 2.411668249999991, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0, + "first_s": 0.034763040999990835, + "last_s": 2.4068437079999967, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "tps": 849.7301738481642, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.370799998009716e-05, + "first_s": 0.08396166599999333, + "last_s": 4.810981999999996, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0007197079999912148, + "first_s": 0.21460262499999772, + "last_s": 4.810679999999991, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00016783299997769063, + "first_s": 0.03487445799999023, + "last_s": 4.810400125000001, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00034124999999107786, + "first_s": 0.051219332999977496, + "last_s": 4.810703082999993, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0005149999999787269, + "first_s": 0.06757191599999146, + "last_s": 4.811300790999979, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0010477909999906387, + "first_s": 0.14932912499997997, + "last_s": 4.810914957999984, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0006081659999779276, + "first_s": 0.23090833300000213, + "last_s": 4.811109208000005, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0008241660000010143, + "first_s": 0.01852233299999284, + "last_s": 4.810266999999982, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00024054099998238598, + "first_s": 0.2472579579999774, + "last_s": 4.82003070799999, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 49, + "accepted": 19, + "acceptance_rate": 0.3877551020408163, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.000561665999981642, + "first_s": 0.13296404099997972, + "last_s": 4.81121458299998, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0002917909999951007, + "first_s": 0.18190724999999475, + "last_s": 4.810565624999981, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0009509999999863794, + "first_s": 0.1656304999999918, + "last_s": 4.810872625000002, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0006541660000038974, + "first_s": 0.10027454100000455, + "last_s": 4.810770290999983, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.000441416000001027, + "first_s": 0.11662024999998266, + "last_s": 4.810614290999979, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00039400000000000546, + "first_s": 0.26357329100000015, + "last_s": 4.820046540999982, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.0, + "first_s": 0.1982730409999931, + "last_s": 4.811157332999983, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 1, + "tps": 260.755637278171, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020178915999991887, + "last_s": 0.9981059999999786, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 4, + "tps": 610.621019530317, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.349999999130887e-05, + "first_s": 0.039391499999993584, + "last_s": 1.690934124999984, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.00023124999998458406, + "first_s": 0.07702774999998496, + "last_s": 1.6909574999999961, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.00016170899999679023, + "first_s": 0.05818437499999618, + "last_s": 1.6909441249999873, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.0, + "first_s": 0.02052695899999435, + "last_s": 1.6909213749999878, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 35, + "accepted": 8, + "acceptance_rate": 0.22857142857142856, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 8, + "tps": 810.8527875896573, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.325000002036177e-05, + "first_s": 0.15296854200002485, + "last_s": 2.536549292000018, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0004953330000034839, + "first_s": 0.13413920800002188, + "last_s": 2.5364608330000067, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0003479169999991427, + "first_s": 0.1153612920000171, + "last_s": 2.5362098750000257, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0001584580000155711, + "first_s": 0.059039708000000246, + "last_s": 2.536182917000019, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00021399999999971442, + "first_s": 0.09662441700001523, + "last_s": 2.536114000000026, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.000270958000015753, + "first_s": 0.04021508300002097, + "last_s": 2.5361284170000147, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00039370800001847783, + "first_s": 0.0778280830000142, + "last_s": 2.5365047920000166, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0, + "first_s": 0.020679542000010542, + "last_s": 2.5360662080000225, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 16, + "tps": 803.4663797204113, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 8.845800002177384e-05, + "first_s": 0.04009933300000057, + "last_s": 5.094765332999998, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0008120410000174161, + "first_s": 0.17176145800002018, + "last_s": 5.094510458000002, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000310708000000659, + "first_s": 0.05898112500000252, + "last_s": 5.094143791000022, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0001566660000094089, + "first_s": 0.24710858299999927, + "last_s": 5.094611000000015, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005229580000047918, + "first_s": 0.20944224999999506, + "last_s": 5.094841625000015, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000207208000006176, + "first_s": 0.02121933300000478, + "last_s": 5.094091125000006, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0002610000000231594, + "first_s": 0.19056554100001222, + "last_s": 5.094526875000014, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005700410000031297, + "first_s": 0.15298312500001998, + "last_s": 5.095028416000019, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007608330000152819, + "first_s": 0.3035269160000098, + "last_s": 5.099216500000011, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0006172909999975218, + "first_s": 0.26590962500000614, + "last_s": 5.094901625000006, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003609580000158985, + "first_s": 0.07779525000000831, + "last_s": 5.094320916000015, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000669291000008343, + "first_s": 0.28474433300002033, + "last_s": 5.0992055410000034, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 35, + "accepted": 3, + "acceptance_rate": 0.08571428571428572, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.000412083000014718, + "first_s": 0.0965630830000066, + "last_s": 5.09512112500002, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0004597080000223741, + "first_s": 0.13415579100001196, + "last_s": 5.095075790999999, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007149580000032074, + "first_s": 0.22830112500000155, + "last_s": 5.094542250000018, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.11536295800001994, + "last_s": 5.094663875000009, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "tps": 357.11403543936194, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.018142707999999175, + "last_s": 0.7322003330000086, + "chunks": 90, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 89, + "drafted": 298, + "accepted": 166, + "acceptance_rate": 0.5570469798657718, + "tokens_per_round": 2.8764044943820224 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "tps": 633.0089663616201, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00011112499998944259, + "first_s": 0.03549404099999265, + "last_s": 1.6050766660000022, + "chunks": 221, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 220, + "drafted": 53, + "accepted": 35, + "acceptance_rate": 0.660377358490566, + "tokens_per_round": 1.1636363636363636 + } + }, + { + "sent_s": 0.00029341600000520884, + "first_s": 0.06967379100001381, + "last_s": 1.6297457910000048, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 55, + "accepted": 33, + "acceptance_rate": 0.6, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0001966250000009495, + "first_s": 0.05259558299999867, + "last_s": 1.6297132910000016, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 56, + "accepted": 33, + "acceptance_rate": 0.5892857142857143, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.018394207999989476, + "last_s": 1.6050416660000053, + "chunks": 221, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 220, + "drafted": 54, + "accepted": 35, + "acceptance_rate": 0.6481481481481481, + "tokens_per_round": 1.1636363636363636 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "tps": 811.9038369967539, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.791599998720812e-05, + "first_s": 0.10382895800000824, + "last_s": 2.496403291000007, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 39, + "accepted": 30, + "acceptance_rate": 0.7692307692307693, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.00037237499998354906, + "first_s": 0.018563124999985803, + "last_s": 2.483367208000004, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 42, + "accepted": 31, + "acceptance_rate": 0.7380952380952381, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00030970800000318377, + "first_s": 0.035656749999986914, + "last_s": 2.4962025410000024, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.00023995800000875533, + "first_s": 0.06971212499999524, + "last_s": 2.496284457999991, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 42, + "accepted": 31, + "acceptance_rate": 0.7380952380952381, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.00017233299999475094, + "first_s": 0.086754749999983, + "last_s": 2.4965703750000046, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 41, + "accepted": 31, + "acceptance_rate": 0.7560975609756098, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.00047766599999476966, + "first_s": 0.05266662500000052, + "last_s": 2.496250916000008, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 45, + "accepted": 31, + "acceptance_rate": 0.6888888888888889, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.000425707999994529, + "first_s": 0.13798900000000458, + "last_s": 2.5311759580000057, + "chunks": 230, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 229, + "drafted": 42, + "accepted": 26, + "acceptance_rate": 0.6190476190476191, + "tokens_per_round": 1.1179039301310043 + } + }, + { + "sent_s": 0.0, + "first_s": 0.12091237499998897, + "last_s": 2.465353166, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 48, + "accepted": 32, + "acceptance_rate": 0.6666666666666666, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "tps": 883.3093581379466, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.099999999762076e-05, + "first_s": 0.018246500000003607, + "last_s": 3.8595087500000034, + "chunks": 173, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 172, + "drafted": 118, + "accepted": 83, + "acceptance_rate": 0.7033898305084746, + "tokens_per_round": 1.4883720930232558 + } + }, + { + "sent_s": 0.000520624999978736, + "first_s": 0.08644045899998787, + "last_s": 4.606953958999981, + "chunks": 215, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 214, + "drafted": 62, + "accepted": 41, + "acceptance_rate": 0.6612903225806451, + "tokens_per_round": 1.1962616822429906 + } + }, + { + "sent_s": 0.0001584999999977299, + "first_s": 0.20598312499998883, + "last_s": 4.626133334000002, + "chunks": 218, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 217, + "drafted": 48, + "accepted": 38, + "acceptance_rate": 0.7916666666666666, + "tokens_per_round": 1.1797235023041475 + } + }, + { + "sent_s": 0.00021029199999134107, + "first_s": 0.17181783399999517, + "last_s": 4.341668666999993, + "chunks": 198, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 197, + "drafted": 74, + "accepted": 58, + "acceptance_rate": 0.7837837837837838, + "tokens_per_round": 1.299492385786802 + } + }, + { + "sent_s": 0.0009690839999905165, + "first_s": 0.27408645899998874, + "last_s": 4.429191208999981, + "chunks": 204, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 203, + "drafted": 62, + "accepted": 52, + "acceptance_rate": 0.8387096774193549, + "tokens_per_round": 1.2610837438423645 + } + }, + { + "sent_s": 0.0005835839999974723, + "first_s": 0.22295670899998754, + "last_s": 4.626142375000001, + "chunks": 218, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 217, + "drafted": 46, + "accepted": 38, + "acceptance_rate": 0.8260869565217391, + "tokens_per_round": 1.1797235023041475 + } + }, + { + "sent_s": 0.0002605839999887394, + "first_s": 0.06939641699997878, + "last_s": 4.598476750000003, + "chunks": 214, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 213, + "drafted": 60, + "accepted": 42, + "acceptance_rate": 0.7, + "tokens_per_round": 1.2018779342723005 + } + }, + { + "sent_s": 0.0006320839999887085, + "first_s": 0.05231562499997722, + "last_s": 4.6068307919999825, + "chunks": 215, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 214, + "drafted": 56, + "accepted": 41, + "acceptance_rate": 0.7321428571428571, + "tokens_per_round": 1.1962616822429906 + } + }, + { + "sent_s": 0.00031029199999466073, + "first_s": 0.25705649999997604, + "last_s": 4.637239792000003, + "chunks": 220, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 219, + "drafted": 45, + "accepted": 37, + "acceptance_rate": 0.8222222222222222, + "tokens_per_round": 1.1689497716894977 + } + }, + { + "sent_s": 0.0008008339999889813, + "first_s": 0.18887137499999085, + "last_s": 4.429128374999976, + "chunks": 204, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 203, + "drafted": 65, + "accepted": 52, + "acceptance_rate": 0.8, + "tokens_per_round": 1.2610837438423645 + } + }, + { + "sent_s": 0.000749458999990793, + "first_s": 0.1376906249999763, + "last_s": 4.189068583999983, + "chunks": 191, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 190, + "drafted": 82, + "accepted": 65, + "acceptance_rate": 0.7926829268292683, + "tokens_per_round": 1.3473684210526315 + } + }, + { + "sent_s": 0.00035833399999773974, + "first_s": 0.03527495899999167, + "last_s": 4.444412958999976, + "chunks": 205, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 204, + "drafted": 83, + "accepted": 51, + "acceptance_rate": 0.6144578313253012, + "tokens_per_round": 1.2549019607843137 + } + }, + { + "sent_s": 0.0007010839999850305, + "first_s": 0.10352499999999054, + "last_s": 4.384193791999991, + "chunks": 201, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 200, + "drafted": 73, + "accepted": 55, + "acceptance_rate": 0.7534246575342466, + "tokens_per_round": 1.28 + } + }, + { + "sent_s": 0.0004536249999773645, + "first_s": 0.23998974999997813, + "last_s": 4.620054084000003, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 51, + "accepted": 39, + "acceptance_rate": 0.7647058823529411, + "tokens_per_round": 1.1851851851851851 + } + }, + { + "sent_s": 0.00040704199997776414, + "first_s": 0.15474045899998146, + "last_s": 4.444532624999994, + "chunks": 205, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 204, + "drafted": 62, + "accepted": 51, + "acceptance_rate": 0.8225806451612904, + "tokens_per_round": 1.2549019607843137 + } + }, + { + "sent_s": 0.0, + "first_s": 0.1206221669999934, + "last_s": 4.3558931669999765, + "chunks": 199, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 198, + "drafted": 75, + "accepted": 57, + "acceptance_rate": 0.76, + "tokens_per_round": 1.292929292929293 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 1, + "tps": 357.13808429747166, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020374333000006573, + "last_s": 0.7343838750000202, + "chunks": 92, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 4, + "tps": 616.2194080955929, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.549999998625935e-05, + "first_s": 0.040322750000001406, + "last_s": 1.6757240409999952, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 16, + "accepted": 6, + "acceptance_rate": 0.375, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00024083299999233532, + "first_s": 0.07914074999999343, + "last_s": 1.675769874999986, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00016987499998322164, + "first_s": 0.020515290999981062, + "last_s": 1.6757372079999868, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0, + "first_s": 0.05971466599999076, + "last_s": 1.6757571659999826, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 8, + "tps": 795.6656503728173, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.137500001088483e-05, + "first_s": 0.07913854200000969, + "last_s": 2.584796124999997, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 5, + "accepted": 2, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.0002701250000143318, + "first_s": 0.020917375000010452, + "last_s": 2.570523709000014, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0004204170000150498, + "first_s": 0.15666095899999277, + "last_s": 2.5707824170000038, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 16, + "accepted": 5, + "acceptance_rate": 0.3125, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0001636670000095819, + "first_s": 0.13726049999999645, + "last_s": 2.5706546670000137, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 16, + "accepted": 5, + "acceptance_rate": 0.3125, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.000354375000000573, + "first_s": 0.1179348750000031, + "last_s": 2.5708748750000154, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 9, + "accepted": 5, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0005322920000025988, + "first_s": 0.09854445900000997, + "last_s": 2.5848083749999944, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 5, + "accepted": 2, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.0002184589999956188, + "first_s": 0.05977058400000601, + "last_s": 2.570543917000009, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0, + "first_s": 0.040353542000019615, + "last_s": 2.570704667000001, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 16, + "tps": 873.8662039536016, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 8.758299998135044e-05, + "first_s": 0.021479124999984833, + "last_s": 4.573498540999992, + "chunks": 240, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 29, + "accepted": 16, + "acceptance_rate": 0.5517241379310345, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00015283299995871857, + "first_s": 0.040829290999965906, + "last_s": 4.685387665999997, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 19, + "accepted": 6, + "acceptance_rate": 0.3157894736842105, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0008949999999572356, + "first_s": 0.2153574579999713, + "last_s": 4.677621249999959, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 15, + "accepted": 7, + "acceptance_rate": 0.4666666666666667, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0005408750000128748, + "first_s": 0.09905920799997148, + "last_s": 4.435126749999995, + "chunks": 230, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 229, + "drafted": 43, + "accepted": 26, + "acceptance_rate": 0.6046511627906976, + "tokens_per_round": 1.1179039301310043 + } + }, + { + "sent_s": 0.0004524579999838352, + "first_s": 0.060291582999980164, + "last_s": 4.690369165999982, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 15, + "accepted": 5, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0007604580000020178, + "first_s": 0.2928932080000095, + "last_s": 4.685423915999991, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.000204957999983435, + "first_s": 0.1572447499999612, + "last_s": 4.450082915999985, + "chunks": 231, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 230, + "drafted": 41, + "accepted": 25, + "acceptance_rate": 0.6097560975609756, + "tokens_per_round": 1.1130434782608696 + } + }, + { + "sent_s": 0.0005966249999573847, + "first_s": 0.31224591599999485, + "last_s": 4.685437250000007, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00025804099999504615, + "first_s": 0.07968004099996051, + "last_s": 4.506589874999975, + "chunks": 235, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 36, + "accepted": 21, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0007083749999878819, + "first_s": 0.2734678329999838, + "last_s": 4.690386082999964, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 12, + "accepted": 5, + "acceptance_rate": 0.4166666666666667, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0003562499999816282, + "first_s": 0.23470574999998917, + "last_s": 4.677773915999978, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 16, + "accepted": 7, + "acceptance_rate": 0.4375, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00030695800001012685, + "first_s": 0.1378970409999738, + "last_s": 3.5230339999999956, + "chunks": 172, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 171, + "drafted": 111, + "accepted": 84, + "acceptance_rate": 0.7567567567567568, + "tokens_per_round": 1.4970760233918128 + } + }, + { + "sent_s": 0.0006464999999593601, + "first_s": 0.25408454099999744, + "last_s": 4.67772920799996, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 15, + "accepted": 7, + "acceptance_rate": 0.4666666666666667, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0010505409999836957, + "first_s": 0.19596420799996395, + "last_s": 4.6674600000000055, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 21, + "accepted": 8, + "acceptance_rate": 0.38095238095238093, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.0004032909999978074, + "first_s": 0.11851341599998477, + "last_s": 4.645478665999974, + "chunks": 246, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 245, + "drafted": 18, + "accepted": 10, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.0448979591836736 + } + }, + { + "sent_s": 0.0, + "first_s": 0.17660416600000417, + "last_s": 4.598797999999988, + "chunks": 242, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 241, + "drafted": 28, + "accepted": 14, + "acceptance_rate": 0.5, + "tokens_per_round": 1.062240663900415 + } + } + ] + }, + { + "round": 1, + "arm": "main", + "prompt": "code", + "streams": 1, + "tps": 290.1445709497884, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.017870000000016262, + "last_s": 0.8967422090000241, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 1, + "arm": "main", + "prompt": "code", + "streams": 4, + "tps": 652.1462364408767, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.841700000379205e-05, + "first_s": 0.03442879199997151, + "last_s": 1.5783031249999908, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0002859169999851474, + "first_s": 0.06710795800000824, + "last_s": 1.582126833000018, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00020595799998091024, + "first_s": 0.050818999999989956, + "last_s": 1.5783118329999866, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.018060457999979462, + "last_s": 1.5782788750000236, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 1, + "arm": "main", + "prompt": "code", + "streams": 8, + "tps": 830.8257532362089, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.612500002731394e-05, + "first_s": 0.0183462080000254, + "last_s": 2.465988500000037, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0004637080000406968, + "first_s": 0.11651212499998564, + "last_s": 2.4664302909999947, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00017512499999838838, + "first_s": 0.13292641600003208, + "last_s": 2.473734707999995, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.00037058300000580857, + "first_s": 0.051134208000007675, + "last_s": 2.466002875000015, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0002378330000283313, + "first_s": 0.034738708000020324, + "last_s": 2.4660924579999914, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00041770799998630537, + "first_s": 0.08384120800002393, + "last_s": 2.4662536250000358, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0002934999999979482, + "first_s": 0.10015804099998604, + "last_s": 2.4664831250000248, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0, + "first_s": 0.06747904100001278, + "last_s": 2.4663362500000403, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + } + ] + }, + { + "round": 1, + "arm": "main", + "prompt": "code", + "streams": 16, + "tps": 830.1942250072074, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 8.816700000124911e-05, + "first_s": 0.035117709000019204, + "last_s": 4.9320778750000045, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00105195899999444, + "first_s": 0.14947258400002283, + "last_s": 4.933139250000011, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00015245899999172252, + "first_s": 0.08419329200000902, + "last_s": 4.932175000000029, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00020837499999970532, + "first_s": 0.10050216700000192, + "last_s": 4.932412166999995, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00042595899998332243, + "first_s": 0.21482041699999854, + "last_s": 4.9328578340000035, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0008812500000203727, + "first_s": 0.24742183400002205, + "last_s": 4.932897459000003, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0005659590000277603, + "first_s": 0.23112195900000643, + "last_s": 4.933094625000024, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00027537500000107684, + "first_s": 0.19848766699999487, + "last_s": 4.932697791999999, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00032454199998710465, + "first_s": 0.16584158400002025, + "last_s": 4.933056958999998, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.000725792000025649, + "first_s": 0.018690292000030695, + "last_s": 4.932056584000009, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0006149999999820466, + "first_s": 0.26380495900002643, + "last_s": 4.93281241699998, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0004815419999886217, + "first_s": 0.11685229199997593, + "last_s": 4.932662250000021, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00037687500002903107, + "first_s": 0.06783812499998021, + "last_s": 4.9330156670000065, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.000664792000009129, + "first_s": 0.18216512499998316, + "last_s": 4.933202917000017, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0008210000000303808, + "first_s": 0.13316987500002142, + "last_s": 4.932942249999996, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0, + "first_s": 0.05148687500002325, + "last_s": 4.932089542000028, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + } + ] + }, + { + "round": 1, + "arm": "main", + "prompt": "prose", + "streams": 1, + "tps": 262.17824367823357, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020563792000018566, + "last_s": 0.9931845419999945, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 1, + "arm": "main", + "prompt": "prose", + "streams": 4, + "tps": 623.428583696316, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.27079999542002e-05, + "first_s": 0.0769434999999703, + "last_s": 1.6566039159999946, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00023712499995554026, + "first_s": 0.05810608299998421, + "last_s": 1.656582125, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00015891599997530648, + "first_s": 0.020490290999987337, + "last_s": 1.65656974999996, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0, + "first_s": 0.03932337499998084, + "last_s": 1.6565929159999655, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 1, + "arm": "main", + "prompt": "prose", + "streams": 8, + "tps": 804.3767029105711, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.345799998072835e-05, + "first_s": 0.0781672499999786, + "last_s": 2.553240374999973, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005230419999975311, + "first_s": 0.13456129200000078, + "last_s": 2.553431499999988, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000334582999983013, + "first_s": 0.05930791700001237, + "last_s": 2.5531172919999676, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0001665830000092683, + "first_s": 0.04046745799996643, + "last_s": 2.5531376249999767, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0004385419999834994, + "first_s": 0.1533778329999791, + "last_s": 2.557074666999995, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002247919999831538, + "first_s": 0.020949499999971977, + "last_s": 2.5531034580000096, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00027691700000787023, + "first_s": 0.11573950000001787, + "last_s": 2.553477832999988, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.09695012499997802, + "last_s": 2.55359920799998, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 1, + "arm": "main", + "prompt": "prose", + "streams": 16, + "tps": 799.5577348294199, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.379199997283649e-05, + "first_s": 0.020883708999974715, + "last_s": 5.122531833999972, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0009131669999646874, + "first_s": 0.2662117919999787, + "last_s": 5.123171499999955, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0001618750000034197, + "first_s": 0.15336545899998555, + "last_s": 5.123476541999992, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00021079199996165698, + "first_s": 0.2473814169999855, + "last_s": 5.123264333999998, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0004082919999746082, + "first_s": 0.059267333999969196, + "last_s": 5.1225703749999525, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0004592919999595324, + "first_s": 0.22860629199999494, + "last_s": 5.123352499999953, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.001210374999971009, + "first_s": 0.28501837499999283, + "last_s": 5.123573999999962, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006401669999718251, + "first_s": 0.04041541699996287, + "last_s": 5.122544374999961, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00026254199997310934, + "first_s": 0.0780780839999693, + "last_s": 5.122579916999996, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005188749999547326, + "first_s": 0.11574570899995251, + "last_s": 5.1237047089999805, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005762089999734599, + "first_s": 0.1721516249999695, + "last_s": 5.123214166999958, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003622919999770602, + "first_s": 0.3038521249999917, + "last_s": 5.1233894169999985, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007301669999719707, + "first_s": 0.20979229199997462, + "last_s": 5.12298283399997, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003171669999915139, + "first_s": 0.13458849999994982, + "last_s": 5.122833166999953, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006837499999505781, + "first_s": 0.09692954199999804, + "last_s": 5.1231065419999595, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.19095787499998096, + "last_s": 5.123651749999965, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "tps": 288.80212241245, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019138124999983575, + "last_s": 0.9020956250000154, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "tps": 662.1169568286863, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00010299999996732367, + "first_s": 0.05080774999998994, + "last_s": 1.5586303339999859, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.00024045899999691756, + "first_s": 0.06717550000001893, + "last_s": 1.5586626670000214, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.00018233399998734967, + "first_s": 0.018149333999986084, + "last_s": 1.5586120840000035, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 63, + "accepted": 37, + "acceptance_rate": 0.5873015873015873, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.0, + "first_s": 0.03449033399999735, + "last_s": 1.5586387920000107, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 63, + "accepted": 37, + "acceptance_rate": 0.5873015873015873, + "tokens_per_round": 1.1743119266055047 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "tps": 851.9150784740538, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.316599999920072e-05, + "first_s": 0.017899332999945727, + "last_s": 2.4075728329999606, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00015991599997278172, + "first_s": 0.13227741599996534, + "last_s": 2.4124968329999774, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0004250409999713156, + "first_s": 0.11592979099998502, + "last_s": 2.412504207999973, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00021020799999860174, + "first_s": 0.034299665999981244, + "last_s": 2.4076144999999656, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0003744579999533926, + "first_s": 0.08332545799999025, + "last_s": 2.4077246249999575, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00026279099995463184, + "first_s": 0.0670137079999904, + "last_s": 2.407703332999972, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.000328249999995478, + "first_s": 0.09966033299997434, + "last_s": 2.407870832999947, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.0506086249999953, + "last_s": 2.407634250000001, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "tps": 849.8077382579876, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.60000000418404e-05, + "first_s": 0.21488825000000134, + "last_s": 4.810995583000022, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0009846660000221164, + "first_s": 0.2311888330000329, + "last_s": 4.810556125000005, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0006709160000468728, + "first_s": 0.10054408300004525, + "last_s": 4.81074537500001, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0004253750000202672, + "first_s": 0.14955316600003243, + "last_s": 4.810960583000053, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00016945800001622047, + "first_s": 0.1168898330000161, + "last_s": 4.810351666000031, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.000221833000011884, + "first_s": 0.018796790999999757, + "last_s": 4.810066791000054, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00027404100001149345, + "first_s": 0.0678322080000271, + "last_s": 4.810110416000043, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0005349160000491793, + "first_s": 0.1985286250000513, + "last_s": 4.81078954100002, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00032766600003242274, + "first_s": 0.08417212500000915, + "last_s": 4.810636458000033, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0004727500000285545, + "first_s": 0.16590491600004498, + "last_s": 4.810835791000045, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0008189580000248498, + "first_s": 0.18221120800001245, + "last_s": 4.810424791000003, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0007290830000101778, + "first_s": 0.05148583300001519, + "last_s": 4.810153916000047, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.000581375000024309, + "first_s": 0.247501375000013, + "last_s": 4.8198596660000135, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 49, + "accepted": 19, + "acceptance_rate": 0.3877551020408163, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.0003775830000449787, + "first_s": 0.03516545800005133, + "last_s": 4.81009825000001, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0006237910000095326, + "first_s": 0.1332368750000228, + "last_s": 4.810877625000046, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0, + "first_s": 0.2638208750000217, + "last_s": 4.819882750000033, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 1, + "tps": 261.43083917633663, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020143625000002885, + "last_s": 0.9955449999999928, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 4, + "tps": 610.2119110482674, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.208300002683245e-05, + "first_s": 0.05877595800001245, + "last_s": 1.6919833750000066, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.00024008299999422889, + "first_s": 0.03996425000002546, + "last_s": 1.691958999999997, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 35, + "accepted": 8, + "acceptance_rate": 0.22857142857142856, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.00016249999998763087, + "first_s": 0.07758241599998428, + "last_s": 1.6919710829999985, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.0, + "first_s": 0.020432916000004298, + "last_s": 1.6919316249999952, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 35, + "accepted": 8, + "acceptance_rate": 0.22857142857142856, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 8, + "tps": 810.1186198318942, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 8.63340000023527e-05, + "first_s": 0.05907720900000868, + "last_s": 2.538353500000028, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0004186670000194681, + "first_s": 0.15300479200004702, + "last_s": 2.5388381250000407, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00015108400003782663, + "first_s": 0.13421100000005026, + "last_s": 2.5387360000000285, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0004976250000368054, + "first_s": 0.0779480840000133, + "last_s": 2.5384655000000294, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0002074170000128106, + "first_s": 0.096706834000031, + "last_s": 2.5384058340000024, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00025504200004888844, + "first_s": 0.02074487500004807, + "last_s": 2.538339334, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00032470900004000214, + "first_s": 0.11547200000001112, + "last_s": 2.5388946250000117, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0, + "first_s": 0.040278709000006074, + "last_s": 2.5383672090000005, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 16, + "tps": 802.9041844105632, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 8.9749999972355e-05, + "first_s": 0.22871504199997617, + "last_s": 5.0983305829999495, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007280419999915466, + "first_s": 0.20994429199998876, + "last_s": 5.098495624999998, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0001584999999977299, + "first_s": 0.04069987499997296, + "last_s": 5.097708082999986, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0002105829999550224, + "first_s": 0.05952895799998714, + "last_s": 5.09855941699999, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0012114999999539577, + "first_s": 0.28504066699997566, + "last_s": 5.1027350829999705, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 35, + "accepted": 3, + "acceptance_rate": 0.08571428571428572, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002629579999506859, + "first_s": 0.1158755419999693, + "last_s": 5.097940249999965, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000671291999992718, + "first_s": 0.09708154199995533, + "last_s": 5.0980255829999805, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007773329999736234, + "first_s": 0.15349004199998717, + "last_s": 5.097905582999999, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006199999999694228, + "first_s": 0.30379487499999414, + "last_s": 5.102750791999995, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 31, + "accepted": 3, + "acceptance_rate": 0.0967741935483871, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00041112499997097984, + "first_s": 0.2474956249999991, + "last_s": 5.098159291999991, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003143749999594547, + "first_s": 0.17228099999999813, + "last_s": 5.098092916999974, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00036224999996647966, + "first_s": 0.26624462499995616, + "last_s": 5.098614749999967, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005654579999827547, + "first_s": 0.1346644999999853, + "last_s": 5.098270832999958, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0009518329999878006, + "first_s": 0.0783074579999834, + "last_s": 5.097780166999996, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00045808299995542257, + "first_s": 0.021197999999969852, + "last_s": 5.097598374999961, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.19109045799996238, + "last_s": 5.098209707999956, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "tps": 357.29358499460534, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.01779850000002625, + "last_s": 0.7314972920000287, + "chunks": 90, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 89, + "drafted": 298, + "accepted": 166, + "acceptance_rate": 0.5570469798657718, + "tokens_per_round": 2.8764044943820224 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "tps": 640.4949813535418, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0001032499999951142, + "first_s": 0.0693399589999899, + "last_s": 1.605799042000001, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 47, + "accepted": 32, + "acceptance_rate": 0.6808510638297872, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00022908400001142581, + "first_s": 0.03524716700002273, + "last_s": 1.6107129170000007, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 50, + "accepted": 31, + "acceptance_rate": 0.62, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00017433400000754773, + "first_s": 0.05225150000001122, + "last_s": 1.6057841250000138, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 48, + "accepted": 32, + "acceptance_rate": 0.6666666666666666, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.018194584000013947, + "last_s": 1.6107005840000284, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 51, + "accepted": 31, + "acceptance_rate": 0.6078431372549019, + "tokens_per_round": 1.1428571428571428 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "tps": 810.8186930899934, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.645800002999749e-05, + "first_s": 0.038159375000020646, + "last_s": 2.49951016600005, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 47, + "accepted": 31, + "acceptance_rate": 0.6595744680851063, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0004860829999984162, + "first_s": 0.12334808299999622, + "last_s": 2.4685120830000074, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 47, + "accepted": 32, + "acceptance_rate": 0.6808510638297872, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00017500000001291482, + "first_s": 0.1062772500000051, + "last_s": 2.4997072500000286, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 39, + "accepted": 30, + "acceptance_rate": 0.7692307692307693, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0003515000000220425, + "first_s": 0.07219908300004363, + "last_s": 2.499624666000045, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 45, + "accepted": 31, + "acceptance_rate": 0.6888888888888889, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0004189160000009906, + "first_s": 0.140390291000017, + "last_s": 2.5344794160000106, + "chunks": 230, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 229, + "drafted": 44, + "accepted": 26, + "acceptance_rate": 0.5909090909090909, + "tokens_per_round": 1.1179039301310043 + } + }, + { + "sent_s": 0.00022383300000683448, + "first_s": 0.055192207999994025, + "last_s": 2.4995872080000368, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0002739580000366004, + "first_s": 0.08921379100002014, + "last_s": 2.4997978330000024, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 41, + "accepted": 31, + "acceptance_rate": 0.7560975609756098, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0, + "first_s": 0.018503875000021708, + "last_s": 2.486560291000046, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 42, + "accepted": 31, + "acceptance_rate": 0.7380952380952381, + "tokens_per_round": 1.1428571428571428 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "tps": 896.9609675675463, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.354200000188939e-05, + "first_s": 0.13778716699999904, + "last_s": 4.558449792000033, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 48, + "accepted": 31, + "acceptance_rate": 0.6458333333333334, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0001641250000261607, + "first_s": 0.22284575000003315, + "last_s": 4.567311417000042, + "chunks": 227, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 226, + "drafted": 44, + "accepted": 29, + "acceptance_rate": 0.6590909090909091, + "tokens_per_round": 1.1327433628318584 + } + }, + { + "sent_s": 0.0007750420000434133, + "first_s": 0.03567437500004189, + "last_s": 4.558373334000009, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 47, + "accepted": 31, + "acceptance_rate": 0.6595744680851063, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00021229200001471327, + "first_s": 0.15483083399999487, + "last_s": 4.032109875000003, + "chunks": 194, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 193, + "drafted": 80, + "accepted": 62, + "acceptance_rate": 0.775, + "tokens_per_round": 1.3264248704663213 + } + }, + { + "sent_s": 0.0005378750000204491, + "first_s": 0.2058414589999984, + "last_s": 4.312854167000012, + "chunks": 210, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 209, + "drafted": 62, + "accepted": 46, + "acceptance_rate": 0.7419354838709677, + "tokens_per_round": 1.2248803827751196 + } + }, + { + "sent_s": 0.0006347089999962918, + "first_s": 0.2398775419999879, + "last_s": 4.220047916999988, + "chunks": 204, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 203, + "drafted": 70, + "accepted": 52, + "acceptance_rate": 0.7428571428571429, + "tokens_per_round": 1.2610837438423645 + } + }, + { + "sent_s": 0.0002655420000223785, + "first_s": 0.1888389590000088, + "last_s": 4.5432662090000235, + "chunks": 223, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 222, + "drafted": 46, + "accepted": 33, + "acceptance_rate": 0.717391304347826, + "tokens_per_round": 1.1531531531531531 + } + }, + { + "sent_s": 0.000589459000025272, + "first_s": 0.2568938340000386, + "last_s": 4.293403000000012, + "chunks": 209, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 208, + "drafted": 63, + "accepted": 47, + "acceptance_rate": 0.746031746031746, + "tokens_per_round": 1.2307692307692308 + } + }, + { + "sent_s": 0.0003151250000428263, + "first_s": 0.05272545899998704, + "last_s": 4.525703000000021, + "chunks": 221, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 220, + "drafted": 50, + "accepted": 35, + "acceptance_rate": 0.7, + "tokens_per_round": 1.1636363636363636 + } + }, + { + "sent_s": 0.0008679169999936676, + "first_s": 0.2739419590000125, + "last_s": 4.515567583999996, + "chunks": 220, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 219, + "drafted": 49, + "accepted": 36, + "acceptance_rate": 0.7346938775510204, + "tokens_per_round": 1.1689497716894977 + } + }, + { + "sent_s": 0.00036395900002617054, + "first_s": 0.12073566700001948, + "last_s": 4.535177583999996, + "chunks": 222, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 221, + "drafted": 52, + "accepted": 34, + "acceptance_rate": 0.6538461538461539, + "tokens_per_round": 1.158371040723982 + } + }, + { + "sent_s": 0.0006935420000218073, + "first_s": 0.08665825000002769, + "last_s": 4.18508970900001, + "chunks": 202, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 201, + "drafted": 78, + "accepted": 54, + "acceptance_rate": 0.6923076923076923, + "tokens_per_round": 1.2736318407960199 + } + }, + { + "sent_s": 0.0004630420000353297, + "first_s": 0.06971445900001072, + "last_s": 4.563319917000001, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 47, + "accepted": 30, + "acceptance_rate": 0.6382978723404256, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0009719590000258904, + "first_s": 0.10370554200000015, + "last_s": 4.501289917000008, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.00041558399999530593, + "first_s": 0.17182487500002708, + "last_s": 4.355611875000022, + "chunks": 212, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 211, + "drafted": 60, + "accepted": 44, + "acceptance_rate": 0.7333333333333333, + "tokens_per_round": 1.2132701421800949 + } + }, + { + "sent_s": 0.0, + "first_s": 0.018618500000002314, + "last_s": 4.558345916999997, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 50, + "accepted": 31, + "acceptance_rate": 0.62, + "tokens_per_round": 1.1428571428571428 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 1, + "tps": 356.86650252228407, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.01999879200002397, + "last_s": 0.7345517079999695, + "chunks": 92, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 4, + "tps": 616.5100175053622, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.00010187499998437488, + "first_s": 0.07893824999996468, + "last_s": 1.6751000419999968, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0002444589999868185, + "first_s": 0.059547833999999966, + "last_s": 1.6750071249999792, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00016962500001227454, + "first_s": 0.020625709000000825, + "last_s": 1.6749675419999903, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0, + "first_s": 0.040122209000003295, + "last_s": 1.6749933749999855, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 8, + "tps": 796.7632013621845, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.295799998199072e-05, + "first_s": 0.13753887500001838, + "last_s": 2.5673007080000048, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 16, + "accepted": 5, + "acceptance_rate": 0.3125, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00043887499998618296, + "first_s": 0.09879179199998589, + "last_s": 2.581307667000033, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 5, + "accepted": 2, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.00016079200003105143, + "first_s": 0.06006941700002244, + "last_s": 2.567203917000029, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00021183299998028815, + "first_s": 0.04066891700000497, + "last_s": 2.566976082999986, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00026820800002269607, + "first_s": 0.11815679200003615, + "last_s": 2.567093457999988, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 9, + "accepted": 5, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0003361670000003869, + "first_s": 0.15692362499999035, + "last_s": 2.567255792000026, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 16, + "accepted": 5, + "acceptance_rate": 0.3125, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0003837919999796213, + "first_s": 0.07947441700002855, + "last_s": 2.5812740000000076, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 5, + "accepted": 2, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.0, + "first_s": 0.020948458000020764, + "last_s": 2.566939208000008, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 16, + "tps": 857.1512455202633, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.354200000188939e-05, + "first_s": 0.09920666699997582, + "last_s": 4.723476749999975, + "chunks": 247, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 246, + "drafted": 22, + "accepted": 9, + "acceptance_rate": 0.4090909090909091, + "tokens_per_round": 1.0406504065040652 + } + }, + { + "sent_s": 0.00015879199997925753, + "first_s": 0.0604097089999982, + "last_s": 4.774901166999996, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 14, + "accepted": 4, + "acceptance_rate": 0.2857142857142857, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007706669999834048, + "first_s": 0.1767391669999938, + "last_s": 4.690660833999971, + "chunks": 244, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 243, + "drafted": 24, + "accepted": 12, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0534979423868314 + } + }, + { + "sent_s": 0.00020845899996402295, + "first_s": 0.07982154199999059, + "last_s": 4.233565124999984, + "chunks": 211, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 210, + "drafted": 66, + "accepted": 45, + "acceptance_rate": 0.6818181818181818, + "tokens_per_round": 1.2190476190476192 + } + }, + { + "sent_s": 0.0010249589999489217, + "first_s": 0.11857479199994714, + "last_s": 4.39235599999995, + "chunks": 222, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 221, + "drafted": 50, + "accepted": 34, + "acceptance_rate": 0.68, + "tokens_per_round": 1.158371040723982 + } + }, + { + "sent_s": 0.0002622089999704258, + "first_s": 0.234921791999966, + "last_s": 4.767315209000003, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 11, + "accepted": 5, + "acceptance_rate": 0.45454545454545453, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0005217919999722653, + "first_s": 0.2930434169999785, + "last_s": 4.774883041999999, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 7, + "accepted": 4, + "acceptance_rate": 0.5714285714285714, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000560291999988749, + "first_s": 0.19614500000000135, + "last_s": 4.781044624999993, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 13, + "accepted": 3, + "acceptance_rate": 0.23076923076923078, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0004085420000023987, + "first_s": 0.021137416999977177, + "last_s": 3.7824979589999543, + "chunks": 182, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 181, + "drafted": 104, + "accepted": 74, + "acceptance_rate": 0.7115384615384616, + "tokens_per_round": 1.4143646408839778 + } + }, + { + "sent_s": 0.0009459999999990032, + "first_s": 0.2543154999999615, + "last_s": 4.767446791999987, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 11, + "accepted": 5, + "acceptance_rate": 0.45454545454545453, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0006212089999735326, + "first_s": 0.13793583399996123, + "last_s": 4.666214541999977, + "chunks": 242, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 241, + "drafted": 24, + "accepted": 14, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.062240663900415 + } + }, + { + "sent_s": 0.00031029199999466073, + "first_s": 0.1573439999999664, + "last_s": 4.678867708999974, + "chunks": 243, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 242, + "drafted": 29, + "accepted": 13, + "acceptance_rate": 0.4482758620689655, + "tokens_per_round": 1.0578512396694215 + } + }, + { + "sent_s": 0.00035820899995542277, + "first_s": 0.3124601249999728, + "last_s": 4.781090833999997, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 12, + "accepted": 3, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.000694291999991492, + "first_s": 0.2155093339999894, + "last_s": 4.781077083999946, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0004538339999839991, + "first_s": 0.2736370839999722, + "last_s": 4.767494124999985, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 9, + "accepted": 5, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0, + "first_s": 0.04105074999995395, + "last_s": 4.173923208999952, + "chunks": 207, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 206, + "drafted": 73, + "accepted": 49, + "acceptance_rate": 0.6712328767123288, + "tokens_per_round": 1.2427184466019416 + } + } + ] + }, + { + "round": 2, + "arm": "main", + "prompt": "code", + "streams": 1, + "tps": 289.230570484437, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.01927791699995396, + "last_s": 0.900927458999945, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 2, + "arm": "main", + "prompt": "code", + "streams": 4, + "tps": 651.5197726117417, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.487500000204818e-05, + "first_s": 0.018004292000000532, + "last_s": 1.5797410420000233, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00025370800000246163, + "first_s": 0.05079220800001849, + "last_s": 1.5797607499999913, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00016833299997642825, + "first_s": 0.03436199999998735, + "last_s": 1.5797775830000091, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.06716679200002318, + "last_s": 1.5835745829999723, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + } + } + ] + }, + { + "round": 2, + "arm": "main", + "prompt": "code", + "streams": 8, + "tps": 829.9116850904256, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.608400000615802e-05, + "first_s": 0.051229583999997885, + "last_s": 2.468868917000009, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0004387089999795535, + "first_s": 0.08406691700002966, + "last_s": 2.4689808340000354, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00017216700001654317, + "first_s": 0.03484574999998813, + "last_s": 2.4688760420000335, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00027787499999476495, + "first_s": 0.11678216699999666, + "last_s": 2.469216249999988, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0003801670000029844, + "first_s": 0.13312216700001045, + "last_s": 2.476538667, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.0004904589999910058, + "first_s": 0.10042066699998031, + "last_s": 2.46926479199999, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0002266249999820502, + "first_s": 0.06765316700000312, + "last_s": 2.4690102090000323, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0, + "first_s": 0.01844579200002272, + "last_s": 2.4688550420000297, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + } + ] + }, + { + "round": 2, + "arm": "main", + "prompt": "code", + "streams": 16, + "tps": 828.7649555945501, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.304200000315177e-05, + "first_s": 0.1984182499999747, + "last_s": 4.941143875000023, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0005334589999961281, + "first_s": 0.16576958399997466, + "last_s": 4.940930334000029, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00016395900001953123, + "first_s": 0.06765066700000943, + "last_s": 4.9407357499999875, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0008591669999873375, + "first_s": 0.05131333399998539, + "last_s": 4.940530000000024, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0002123749999896063, + "first_s": 0.018543667000017194, + "last_s": 4.940416625000012, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0002662499999814827, + "first_s": 0.24748233399998298, + "last_s": 4.941041874999996, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0005948749999902248, + "first_s": 0.11670366700002432, + "last_s": 4.941234375000022, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00031779199997572505, + "first_s": 0.1003815839999902, + "last_s": 4.940843374999986, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0009180420000234335, + "first_s": 0.18208241700000372, + "last_s": 4.941531750000024, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0006782500000213076, + "first_s": 0.08403616700002203, + "last_s": 4.940987167000003, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00036941700000170385, + "first_s": 0.03497149999998328, + "last_s": 4.940430166999988, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0004694169999766018, + "first_s": 0.21478429199999027, + "last_s": 4.941419459000031, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0007815839999807395, + "first_s": 0.23112329200000659, + "last_s": 4.941494583999997, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0009819169999900623, + "first_s": 0.26384345899998607, + "last_s": 4.941331084000012, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.00041783400001804694, + "first_s": 0.14942258399997854, + "last_s": 4.941079417000026, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + }, + { + "sent_s": 0.0, + "first_s": 0.13306408400001146, + "last_s": 4.941375124999979, + "chunks": 240, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + } + } + ] + }, + { + "round": 2, + "arm": "main", + "prompt": "prose", + "streams": 1, + "tps": 262.4367240358528, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020232375000034608, + "last_s": 0.9918951669999956, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 2, + "arm": "main", + "prompt": "prose", + "streams": 4, + "tps": 622.7480552674332, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.020900000678012e-05, + "first_s": 0.0393185840000001, + "last_s": 1.658318917000031, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00021545900000319307, + "first_s": 0.02044062500004884, + "last_s": 1.658307667000031, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0001620840000100543, + "first_s": 0.058112750000020696, + "last_s": 1.658328417000007, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0, + "first_s": 0.0769772500000272, + "last_s": 1.6583421670000007, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 2, + "arm": "main", + "prompt": "prose", + "streams": 8, + "tps": 803.714540097247, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 8.941600003709027e-05, + "first_s": 0.09670395800003462, + "last_s": 2.5554197499999987, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003854580000393071, + "first_s": 0.11549291599999378, + "last_s": 2.5553644160000317, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00025637500004904723, + "first_s": 0.13432833300004177, + "last_s": 2.5555472080000072, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00020395800004280318, + "first_s": 0.07788545800002566, + "last_s": 2.5549879580000265, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00015408300004082776, + "first_s": 0.020731458000000202, + "last_s": 2.554933833000007, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0004990410000118572, + "first_s": 0.15311308299999382, + "last_s": 2.5589460830000235, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00032250000003841706, + "first_s": 0.04028633300004003, + "last_s": 2.5549463750000427, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.05905616600000485, + "last_s": 2.554976666000016, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 2, + "arm": "main", + "prompt": "prose", + "streams": 16, + "tps": 797.5772223478886, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.662500002605157e-05, + "first_s": 0.21002141600001778, + "last_s": 5.1364576660000125, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007576250000056461, + "first_s": 0.24770233300000655, + "last_s": 5.1359366660000205, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00016866600003595522, + "first_s": 0.07852545800000144, + "last_s": 5.1356096659999935, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0008556660000067495, + "first_s": 0.1537516250000408, + "last_s": 5.1361924160000285, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0002256250000414184, + "first_s": 0.17250141600004554, + "last_s": 5.13665770800003, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005367080000269198, + "first_s": 0.2665476660000081, + "last_s": 5.136063791000026, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0002780410000013944, + "first_s": 0.0211655830000268, + "last_s": 5.135473832999992, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005808330000149908, + "first_s": 0.28539037500001996, + "last_s": 5.136316791000013, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00033262499999864303, + "first_s": 0.09731941600000482, + "last_s": 5.136251458000004, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003818750000164073, + "first_s": 0.3042443749999961, + "last_s": 5.136134208000044, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0009234999999989668, + "first_s": 0.22887383300002284, + "last_s": 5.136615790999997, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0010432910000304219, + "first_s": 0.11616412500001161, + "last_s": 5.136565625000003, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000431083000023591, + "first_s": 0.19128108300003532, + "last_s": 5.135753458000011, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006715830000416645, + "first_s": 0.1349400830000036, + "last_s": 5.136520415999996, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00047654100001182087, + "first_s": 0.04082654100000127, + "last_s": 5.135484958000006, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.0596649160000311, + "last_s": 5.135501207999994, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 1, + "tps": 355.76141662801433, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019272375000014108, + "last_s": 0.7360448750000046, + "chunks": 90, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 89, + "drafted": 298, + "accepted": 166, + "acceptance_rate": 0.5570469798657718, + "tokens_per_round": 2.8764044943820224 + } + } + ] + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 4, + "tps": 645.2786322623483, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00010016699997095202, + "first_s": 0.01819808400000511, + "last_s": 1.5950289589999898, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0002579169999989972, + "first_s": 0.05222191699999712, + "last_s": 1.5989105840000093, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00017329199999949196, + "first_s": 0.0692620419999912, + "last_s": 1.5889688340000134, + "chunks": 223, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 222, + "drafted": 40, + "accepted": 33, + "acceptance_rate": 0.825, + "tokens_per_round": 1.1531531531531531 + } + }, + { + "sent_s": 0.0, + "first_s": 0.03522750000001906, + "last_s": 1.5950382089999948, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 8, + "tps": 809.6644164246909, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00010037499998816202, + "first_s": 0.0355080409999573, + "last_s": 2.522695915999975, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0004983329999959096, + "first_s": 0.12070795799996858, + "last_s": 2.52311737499997, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 29, + "accepted": 22, + "acceptance_rate": 0.7586206896551724, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0003972499999918, + "first_s": 0.05252608299997519, + "last_s": 2.522637457999963, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0003458749999936117, + "first_s": 0.06958604099997956, + "last_s": 2.5229750830000057, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.00017412500000091313, + "first_s": 0.018461415999979636, + "last_s": 2.5226144999999747, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0002841250000074069, + "first_s": 0.08661762499997394, + "last_s": 2.5230261250000012, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 28, + "accepted": 23, + "acceptance_rate": 0.8214285714285714, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.00023033299999042356, + "first_s": 0.1377665829999728, + "last_s": 2.5380237910000005, + "chunks": 238, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 237, + "drafted": 24, + "accepted": 18, + "acceptance_rate": 0.75, + "tokens_per_round": 1.080168776371308 + } + }, + { + "sent_s": 0.0, + "first_s": 0.1036599159999696, + "last_s": 2.523074832999953, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + } + } + ] + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 16, + "tps": 799.0692280251305, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.429200002841753e-05, + "first_s": 0.035924374999979136, + "last_s": 5.123591916999999, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00015887500001099397, + "first_s": 0.27468241700000817, + "last_s": 5.124402667000027, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0008406670000340455, + "first_s": 0.20647437500002752, + "last_s": 5.124251458999993, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002103749999946558, + "first_s": 0.087138374999995, + "last_s": 5.124144584000021, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0005356249999977081, + "first_s": 0.1893902090000097, + "last_s": 5.124441833999981, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002624169999876358, + "first_s": 0.25761429200002794, + "last_s": 5.124718541999982, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.000746375000005628, + "first_s": 0.07004283399999167, + "last_s": 5.124126208999996, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.000585166999997, + "first_s": 0.15526004200000898, + "last_s": 5.1245976249999785, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00031033400000524125, + "first_s": 0.22353983400000743, + "last_s": 5.124325959000032, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0007956670000339727, + "first_s": 0.052985625000019354, + "last_s": 5.124160416999985, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0006322090000026037, + "first_s": 0.17232549999999947, + "last_s": 5.12402079200001, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0003609999999980573, + "first_s": 0.01883658399998467, + "last_s": 5.123523834000025, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0006907920000003287, + "first_s": 0.24057662500001697, + "last_s": 5.124557625000023, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0004604169999993246, + "first_s": 0.12116666700001133, + "last_s": 5.124777166999991, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00040904200000113633, + "first_s": 0.1382068749999803, + "last_s": 5.124481709000008, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0, + "first_s": 0.10419675000002826, + "last_s": 5.123928625000019, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + } + ] + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 1, + "tps": 359.5743061551645, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.02006162499998254, + "last_s": 0.7292335419999745, + "chunks": 91, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 90, + "drafted": 280, + "accepted": 165, + "acceptance_rate": 0.5892857142857143, + "tokens_per_round": 2.8444444444444446 + } + } + ] + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 4, + "tps": 617.636426500965, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.00010516700001517165, + "first_s": 0.04048800000003894, + "last_s": 1.6718915420000258, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00024808400002029884, + "first_s": 0.0793447500000184, + "last_s": 1.6719714590000194, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00018512500002998422, + "first_s": 0.0598595840000371, + "last_s": 1.6721314590000134, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0, + "first_s": 0.02067445900001985, + "last_s": 1.671916500000009, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + } + ] + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 8, + "tps": 787.7309614581078, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.766700003410733e-05, + "first_s": 0.09887779199999613, + "last_s": 2.610538833000021, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 6, + "accepted": 2, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.00038387500001135777, + "first_s": 0.07951358299999356, + "last_s": 2.5942442080000205, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 8, + "accepted": 5, + "acceptance_rate": 0.625, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00016558300001179305, + "first_s": 0.02083362500002295, + "last_s": 2.5941928330000223, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 15, + "accepted": 5, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0005785420000279373, + "first_s": 0.15703295800000205, + "last_s": 2.6008843330000104, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0002677919999882761, + "first_s": 0.13771400000001677, + "last_s": 2.6008761250000134, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 12, + "accepted": 4, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00021741699998756303, + "first_s": 0.040725208000026214, + "last_s": 2.594233250000002, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 9, + "accepted": 5, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0004375830000071801, + "first_s": 0.06012770800003864, + "last_s": 2.5942115000000285, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 8, + "accepted": 5, + "acceptance_rate": 0.625, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0, + "first_s": 0.11830204200003891, + "last_s": 2.6105502920000276, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 6, + "accepted": 2, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.0118577075098814 + } + } + ] + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 16, + "tps": 792.7530903708569, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.00010041699999874254, + "first_s": 0.23493779199998244, + "last_s": 5.167224250000004, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0005097499999919819, + "first_s": 0.021199499999966065, + "last_s": 5.166656958999965, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00016687499999079591, + "first_s": 0.3124321249999866, + "last_s": 5.167750834000003, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00044941699997025353, + "first_s": 0.04103566700001693, + "last_s": 5.167538583999999, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0005593340000018543, + "first_s": 0.21554237499998408, + "last_s": 5.167156292000016, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0002186670000128288, + "first_s": 0.11859137499999406, + "last_s": 5.167411125000001, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0007038749999992433, + "first_s": 0.15737291699997513, + "last_s": 5.167820875000018, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00026816700000154015, + "first_s": 0.06046645899999703, + "last_s": 5.166726417000007, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.000315875000012511, + "first_s": 0.17679329200001348, + "last_s": 5.167362916999991, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0008672089999777199, + "first_s": 0.25429754200001753, + "last_s": 5.167245292000018, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0007588749999740685, + "first_s": 0.2930327920000195, + "last_s": 5.167315083999995, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0003598750000151085, + "first_s": 0.13801645899997084, + "last_s": 5.167700583999988, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0006246250000003783, + "first_s": 0.2736417090000032, + "last_s": 5.167455874999973, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0008143750000044747, + "first_s": 0.19622391699999753, + "last_s": 5.1670614170000135, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00040524999997160194, + "first_s": 0.07984820899997658, + "last_s": 5.166772917000003, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0, + "first_s": 0.09918999999996458, + "last_s": 5.167650292000019, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "tps": 355.56191780544566, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.01761995799995475, + "last_s": 0.7347946249999495, + "chunks": 90, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 89, + "drafted": 301, + "accepted": 166, + "acceptance_rate": 0.5514950166112956, + "tokens_per_round": 2.8764044943820224 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "tps": 633.0814868973765, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.78340000301614e-05, + "first_s": 0.03571450000004006, + "last_s": 1.6046670000000063, + "chunks": 221, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 220, + "drafted": 54, + "accepted": 35, + "acceptance_rate": 0.6481481481481481, + "tokens_per_round": 1.1636363636363636 + } + }, + { + "sent_s": 0.0002466250000452419, + "first_s": 0.06973966700002165, + "last_s": 1.629312584000047, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 54, + "accepted": 33, + "acceptance_rate": 0.6111111111111112, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00017150000002175148, + "first_s": 0.018145584000023973, + "last_s": 1.6046562920000156, + "chunks": 221, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 220, + "drafted": 54, + "accepted": 35, + "acceptance_rate": 0.6481481481481481, + "tokens_per_round": 1.1636363636363636 + } + }, + { + "sent_s": 0.0, + "first_s": 0.05273579200002132, + "last_s": 1.629299250000031, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 55, + "accepted": 33, + "acceptance_rate": 0.6, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "tps": 813.2946971840239, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.820799999715746e-05, + "first_s": 0.018363375000035376, + "last_s": 2.491688291999992, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 42, + "accepted": 31, + "acceptance_rate": 0.7380952380952381, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.00041337499999372085, + "first_s": 0.13768166700003803, + "last_s": 2.5266792500000292, + "chunks": 230, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 229, + "drafted": 41, + "accepted": 26, + "acceptance_rate": 0.6341463414634146, + "tokens_per_round": 1.1179039301310043 + } + }, + { + "sent_s": 0.00016579200001842764, + "first_s": 0.05280904200003533, + "last_s": 2.4917732080000405, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 44, + "accepted": 31, + "acceptance_rate": 0.7045454545454546, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0005046250000191321, + "first_s": 0.1038404170000149, + "last_s": 2.4918392079999876, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 42, + "accepted": 30, + "acceptance_rate": 0.7142857142857143, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.00021883300001945827, + "first_s": 0.06984158300002719, + "last_s": 2.4920794999999885, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 44, + "accepted": 31, + "acceptance_rate": 0.7045454545454546, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0002707920000375452, + "first_s": 0.03581504200002428, + "last_s": 2.491924332999986, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 45, + "accepted": 30, + "acceptance_rate": 0.6666666666666666, + "tokens_per_round": 1.1377777777777778 + } + }, + { + "sent_s": 0.0003552919999947335, + "first_s": 0.1207515419999936, + "last_s": 2.4611360000000104, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 44, + "accepted": 32, + "acceptance_rate": 0.7272727272727273, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.08679779200002713, + "last_s": 2.4920344170000135, + "chunks": 226, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 225, + "drafted": 42, + "accepted": 30, + "acceptance_rate": 0.7142857142857143, + "tokens_per_round": 1.1377777777777778 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "tps": 886.8595214789708, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.116699999367484e-05, + "first_s": 0.03567537499998252, + "last_s": 4.528998417000025, + "chunks": 218, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 217, + "drafted": 59, + "accepted": 38, + "acceptance_rate": 0.6440677966101694, + "tokens_per_round": 1.1797235023041475 + } + }, + { + "sent_s": 0.0008545840000238059, + "first_s": 0.15475083400002632, + "last_s": 4.249944583999991, + "chunks": 203, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 202, + "drafted": 86, + "accepted": 53, + "acceptance_rate": 0.6162790697674418, + "tokens_per_round": 1.2673267326732673 + } + }, + { + "sent_s": 0.000156917000026624, + "first_s": 0.052642208999998275, + "last_s": 4.481628124999986, + "chunks": 215, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 214, + "drafted": 60, + "accepted": 41, + "acceptance_rate": 0.6833333333333333, + "tokens_per_round": 1.1962616822429906 + } + }, + { + "sent_s": 0.0005277920000139602, + "first_s": 0.2738892090000036, + "last_s": 4.615273375000015, + "chunks": 227, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 226, + "drafted": 40, + "accepted": 29, + "acceptance_rate": 0.725, + "tokens_per_round": 1.1327433628318584 + } + }, + { + "sent_s": 0.00045462500003168316, + "first_s": 0.2058160839999914, + "last_s": 4.5912452090000215, + "chunks": 223, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 222, + "drafted": 53, + "accepted": 33, + "acceptance_rate": 0.6226415094339622, + "tokens_per_round": 1.1531531531531531 + } + }, + { + "sent_s": 0.0002026669999963815, + "first_s": 0.171816917000001, + "last_s": 4.61525745900002, + "chunks": 227, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 226, + "drafted": 44, + "accepted": 29, + "acceptance_rate": 0.6590909090909091, + "tokens_per_round": 1.1327433628318584 + } + }, + { + "sent_s": 0.0010439999999789507, + "first_s": 0.25688379199999645, + "last_s": 4.481758458999991, + "chunks": 215, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 214, + "drafted": 53, + "accepted": 41, + "acceptance_rate": 0.7735849056603774, + "tokens_per_round": 1.1962616822429906 + } + }, + { + "sent_s": 0.0005842090000101052, + "first_s": 0.01870875000003025, + "last_s": 4.214412334000031, + "chunks": 201, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 200, + "drafted": 84, + "accepted": 55, + "acceptance_rate": 0.6547619047619048, + "tokens_per_round": 1.28 + } + }, + { + "sent_s": 0.00026120900002979397, + "first_s": 0.08659845900001528, + "last_s": 4.412412625000002, + "chunks": 211, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 210, + "drafted": 68, + "accepted": 45, + "acceptance_rate": 0.6617647058823529, + "tokens_per_round": 1.2190476190476192 + } + }, + { + "sent_s": 0.0003603339999926902, + "first_s": 0.1377227500000231, + "last_s": 4.619211875000019, + "chunks": 228, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 227, + "drafted": 47, + "accepted": 28, + "acceptance_rate": 0.5957446808510638, + "tokens_per_round": 1.1277533039647578 + } + }, + { + "sent_s": 0.00030870900002355484, + "first_s": 0.2228808749999871, + "last_s": 4.23052958400001, + "chunks": 202, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 201, + "drafted": 68, + "accepted": 54, + "acceptance_rate": 0.7941176470588235, + "tokens_per_round": 1.2736318407960199 + } + }, + { + "sent_s": 0.0007665840000186108, + "first_s": 0.23986387500002593, + "last_s": 4.0215202499999805, + "chunks": 191, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 190, + "drafted": 83, + "accepted": 66, + "acceptance_rate": 0.7951807228915663, + "tokens_per_round": 1.3473684210526315 + } + }, + { + "sent_s": 0.0006267090000164899, + "first_s": 0.069626583999991, + "last_s": 4.5292172919999985, + "chunks": 218, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 217, + "drafted": 51, + "accepted": 38, + "acceptance_rate": 0.7450980392156863, + "tokens_per_round": 1.1797235023041475 + } + }, + { + "sent_s": 0.0004082920000314516, + "first_s": 0.10361899999998059, + "last_s": 4.591206166999996, + "chunks": 223, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 222, + "drafted": 48, + "accepted": 33, + "acceptance_rate": 0.6875, + "tokens_per_round": 1.1531531531531531 + } + }, + { + "sent_s": 0.0006866250000143737, + "first_s": 0.18879587499998252, + "last_s": 4.5630358750000255, + "chunks": 220, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 219, + "drafted": 51, + "accepted": 36, + "acceptance_rate": 0.7058823529411765, + "tokens_per_round": 1.1689497716894977 + } + }, + { + "sent_s": 0.0, + "first_s": 0.12069558400003189, + "last_s": 4.514597999999978, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 60, + "accepted": 39, + "acceptance_rate": 0.65, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 1, + "tps": 356.75662063341406, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020067166999979236, + "last_s": 0.7348401669999589, + "chunks": 92, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 4, + "tps": 615.2096412258015, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.00010383299996874484, + "first_s": 0.04034483300000602, + "last_s": 1.678376582999988, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0002540830000157257, + "first_s": 0.07913179199999831, + "last_s": 1.6784027079999646, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00017783300000928648, + "first_s": 0.059760707999998885, + "last_s": 1.6784122079999975, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0, + "first_s": 0.02044079199998805, + "last_s": 1.6783869170000116, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 8, + "tps": 796.9578088973333, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.466599999541359e-05, + "first_s": 0.0795916659999989, + "last_s": 2.580623707999962, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 5, + "accepted": 2, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0118577075098814 + } + }, + { + "sent_s": 0.00016291599996520745, + "first_s": 0.02094016600000259, + "last_s": 2.5663399579999577, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0003404159999718104, + "first_s": 0.06019687499997417, + "last_s": 2.566447832999984, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0005412079999587149, + "first_s": 0.13779554100000269, + "last_s": 2.566840791000004, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 16, + "accepted": 5, + "acceptance_rate": 0.3125, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00021995800000240706, + "first_s": 0.040804041000001234, + "last_s": 2.5664145000000076, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0004180409999889889, + "first_s": 0.15715837499999452, + "last_s": 2.566752249999979, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 16, + "accepted": 5, + "acceptance_rate": 0.3125, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.00027199999999538704, + "first_s": 0.11837312500000507, + "last_s": 2.5667976659999567, + "chunks": 251, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 250, + "drafted": 9, + "accepted": 5, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.024 + } + }, + { + "sent_s": 0.0, + "first_s": 0.09895945799996753, + "last_s": 2.5806741659999943, + "chunks": 254, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 253, + "drafted": 5, + "accepted": 2, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0118577075098814 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-drafted", + "prompt": "prose", + "streams": 16, + "tps": 854.4605435798837, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 8.824999997614213e-05, + "first_s": 0.02114387500000703, + "last_s": 4.769526541999994, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 16, + "accepted": 6, + "acceptance_rate": 0.375, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00040862499997729174, + "first_s": 0.09924091699997462, + "last_s": 4.016953583999964, + "chunks": 196, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 195, + "drafted": 72, + "accepted": 60, + "acceptance_rate": 0.8333333333333334, + "tokens_per_round": 1.3128205128205128 + } + }, + { + "sent_s": 0.0009741669999812075, + "first_s": 0.15741904199995815, + "last_s": 4.749796083999968, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 15, + "accepted": 7, + "acceptance_rate": 0.4666666666666667, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00015629199998556942, + "first_s": 0.312493083999982, + "last_s": 4.795797708999999, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 5, + "acceptance_rate": 0.5555555555555556, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005290419999823825, + "first_s": 0.2930928339999923, + "last_s": 4.796028833999969, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0002106249999656029, + "first_s": 0.21556441699999596, + "last_s": 4.79570874999996, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 12, + "accepted": 5, + "acceptance_rate": 0.4166666666666667, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0003591249999885804, + "first_s": 0.1186404999999695, + "last_s": 4.3597545839999725, + "chunks": 219, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 58, + "accepted": 37, + "acceptance_rate": 0.6379310344827587, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.0008440839999934724, + "first_s": 0.13799995899995565, + "last_s": 4.562454749999972, + "chunks": 234, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 34, + "accepted": 22, + "acceptance_rate": 0.6470588235294118, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0006364999999846077, + "first_s": 0.27367270899998175, + "last_s": 4.7960864169999695, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 10, + "accepted": 5, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0002579169999989972, + "first_s": 0.25427770900000723, + "last_s": 4.795861041999956, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 5, + "acceptance_rate": 0.45454545454545453, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006784589999710988, + "first_s": 0.23492499999997563, + "last_s": 4.795682749999969, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 5, + "acceptance_rate": 0.45454545454545453, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00030837499997460327, + "first_s": 0.17680974999996124, + "last_s": 4.648196208999991, + "chunks": 241, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 240, + "drafted": 29, + "accepted": 15, + "acceptance_rate": 0.5172413793103449, + "tokens_per_round": 1.0666666666666667 + } + }, + { + "sent_s": 0.0005799999999567262, + "first_s": 0.06045579199997064, + "last_s": 3.590620458999979, + "chunks": 169, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 168, + "drafted": 123, + "accepted": 87, + "acceptance_rate": 0.7073170731707317, + "tokens_per_round": 1.5238095238095237 + } + }, + { + "sent_s": 0.0007273749999967549, + "first_s": 0.04103145900000982, + "last_s": 4.769692292000002, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 15, + "accepted": 6, + "acceptance_rate": 0.4, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.000459708999983377, + "first_s": 0.07988737499999843, + "last_s": 4.402278833999958, + "chunks": 222, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 221, + "drafted": 51, + "accepted": 34, + "acceptance_rate": 0.6666666666666666, + "tokens_per_round": 1.158371040723982 + } + }, + { + "sent_s": 0.0, + "first_s": 0.19619170900000427, + "last_s": 4.795631083999979, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 14, + "accepted": 5, + "acceptance_rate": 0.35714285714285715, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 3, + "arm": "main", + "prompt": "code", + "streams": 1, + "tps": 288.0685129679525, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019108499999958894, + "last_s": 0.904314582999973, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 3, + "arm": "main", + "prompt": "code", + "streams": 4, + "tps": 652.4009218675697, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00010487499997680061, + "first_s": 0.0179946669999822, + "last_s": 1.5776035000000093, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00017887500001734224, + "first_s": 0.06711383399999704, + "last_s": 1.5814504589999956, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0002606250000098953, + "first_s": 0.03437545899998895, + "last_s": 1.5775790000000143, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.050728042000002915, + "last_s": 1.5775919999999815, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 3, + "arm": "main", + "prompt": "code", + "streams": 8, + "tps": 831.6126825307226, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.912500001973967e-05, + "first_s": 0.025430374999984906, + "last_s": 2.47072754200002, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00045554199999742195, + "first_s": 0.09088358399998242, + "last_s": 2.471106458999998, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00035112500000877844, + "first_s": 0.1072000839999987, + "last_s": 2.4711619999999925, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0002247500000294167, + "first_s": 0.07449875000003203, + "last_s": 2.471292917000028, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00017095900000185793, + "first_s": 0.13993212500002983, + "last_s": 2.4784954170000333, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.0005192919999785772, + "first_s": 0.12356725000000779, + "last_s": 2.471208625000031, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0002811250000149812, + "first_s": 0.041804958999989594, + "last_s": 2.470745749999992, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0, + "first_s": 0.058155791999979556, + "last_s": 2.4708504170000083, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + } + ] + }, + { + "round": 3, + "arm": "main", + "prompt": "code", + "streams": 16, + "tps": 830.0623561457346, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.433299999273004e-05, + "first_s": 0.05835183300001745, + "last_s": 4.939848499999982, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.0008314579999932903, + "first_s": 0.1728316250000148, + "last_s": 4.940065875000016, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.0007320000000277105, + "first_s": 0.12382362499999999, + "last_s": 4.940850375000025, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.0012377919999835285, + "first_s": 0.2546014999999784, + "last_s": 4.940907750000008, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.00016191700001400022, + "first_s": 0.02561437500003194, + "last_s": 4.939799875000006, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.00042754200001127174, + "first_s": 0.0911155000000008, + "last_s": 4.94076445799999, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.00021383300003208205, + "first_s": 0.22193470799999204, + "last_s": 4.940359833000002, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.00026975000002948946, + "first_s": 0.1074880830000211, + "last_s": 4.940171292000002, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.0009474999999952161, + "first_s": 0.1892031670000165, + "last_s": 4.9407295419999855, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.0007826670000099512, + "first_s": 0.07470058299998072, + "last_s": 4.939863457999991, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.0005763330000263522, + "first_s": 0.20557658299998138, + "last_s": 4.940600042000028, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.00047649999999066495, + "first_s": 0.04199012499998389, + "last_s": 4.9398321250000095, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.00032729200000858327, + "first_s": 0.15649383300001318, + "last_s": 4.940804249999985, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.00037887500002398156, + "first_s": 0.1401767080000127, + "last_s": 4.940281874999982, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.0010184999999864885, + "first_s": 0.2709963330000278, + "last_s": 4.940641292000009, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + }, + { + "sent_s": 0.0, + "first_s": 0.2382644580000033, + "last_s": 4.940421125, + "chunks": 239, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + } + } + ] + }, + { + "round": 3, + "arm": "main", + "prompt": "prose", + "streams": 1, + "tps": 261.7083370911104, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020360124999967866, + "last_s": 0.9947272499999826, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 3, + "arm": "main", + "prompt": "prose", + "streams": 4, + "tps": 622.3375716098092, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.00010224999999763895, + "first_s": 0.027587374999995973, + "last_s": 1.666514415999984, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00025445799997214635, + "first_s": 0.08412108300001364, + "last_s": 1.666569250000009, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0001738330000193855, + "first_s": 0.06530154099999663, + "last_s": 1.666530458000011, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0, + "first_s": 0.04645841600000722, + "last_s": 1.66653974999997, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 3, + "arm": "main", + "prompt": "prose", + "streams": 8, + "tps": 804.5208417575939, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.104200000820128e-05, + "first_s": 0.020772958000009112, + "last_s": 2.5524102919999905, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00031758300002593387, + "first_s": 0.11569991700002902, + "last_s": 2.5526911670000345, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00021054199999070988, + "first_s": 0.1345285419999982, + "last_s": 2.5528591249999977, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00015941700002031212, + "first_s": 0.04024687500003665, + "last_s": 2.5524276670000177, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00026524999998400745, + "first_s": 0.09682808299999124, + "last_s": 2.5529229580000106, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00046179200001006393, + "first_s": 0.07796595799999295, + "last_s": 2.5530045830000176, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00039845800000648524, + "first_s": 0.05911545800000795, + "last_s": 2.552473833000022, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.1533286669999825, + "last_s": 2.556443750000028, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + } + } + ] + }, + { + "round": 3, + "arm": "main", + "prompt": "prose", + "streams": 16, + "tps": 797.6543488046647, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.82920000183185e-05, + "first_s": 0.05991379200003166, + "last_s": 5.135538708000013, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007642920000421327, + "first_s": 0.28572975000003, + "last_s": 5.136203917000046, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00032704200003763617, + "first_s": 0.0975009170000476, + "last_s": 5.135654750000015, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005441250000330911, + "first_s": 0.1539891250000096, + "last_s": 5.13605429200004, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00017058299999916926, + "first_s": 0.11633641700001363, + "last_s": 5.136342542000023, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0009615000000167129, + "first_s": 0.17281345800000736, + "last_s": 5.135958791999997, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00022329200004378436, + "first_s": 0.21046737500000745, + "last_s": 5.136289792000014, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00042820800001663883, + "first_s": 0.22928483300000835, + "last_s": 5.136428667000018, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0002809169999977712, + "first_s": 0.1351456670000175, + "last_s": 5.136243083000011, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00047337500001276567, + "first_s": 0.26692425000004505, + "last_s": 5.135901042, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006765830000290407, + "first_s": 0.3047265420000258, + "last_s": 5.136158792000003, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005874580000408969, + "first_s": 0.2481052919999911, + "last_s": 5.1363874580000015, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00037908300004119155, + "first_s": 0.041061291999994864, + "last_s": 5.135998167000025, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0006305420000103368, + "first_s": 0.0787272919999964, + "last_s": 5.135376792000045, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007224999999948523, + "first_s": 0.1916815000000156, + "last_s": 5.1358073330000025, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0, + "first_s": 0.021431167000002915, + "last_s": 5.13525258300001, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 1, + "tps": 355.3981583058367, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019139250000023367, + "last_s": 0.7366443750000258, + "chunks": 90, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 89, + "drafted": 301, + "accepted": 166, + "acceptance_rate": 0.5514950166112956, + "tokens_per_round": 2.8764044943820224 + } + } + ] + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 4, + "tps": 645.6944342132804, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0001023330000293754, + "first_s": 0.07652729200003705, + "last_s": 1.595149417000016, + "chunks": 223, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 222, + "drafted": 41, + "accepted": 33, + "acceptance_rate": 0.8048780487804879, + "tokens_per_round": 1.1531531531531531 + } + }, + { + "sent_s": 0.00023770800004285775, + "first_s": 0.025366125000005013, + "last_s": 1.6011396250000303, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00017637500002365414, + "first_s": 0.04245066700002553, + "last_s": 1.6011511250000012, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.05945966700005556, + "last_s": 1.6050607080000532, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + } + ] + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 8, + "tps": 817.5820896934242, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.62500000127875e-05, + "first_s": 0.03566116700000066, + "last_s": 2.4972394590000135, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0002860840000380449, + "first_s": 0.018587209000031635, + "last_s": 2.497224417000041, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0004294590000313292, + "first_s": 0.12091908400003604, + "last_s": 2.5022188340000184, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00017379199999822958, + "first_s": 0.05266087500001504, + "last_s": 2.497260959000016, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.00022987500000226646, + "first_s": 0.13794095899999093, + "last_s": 2.513749500000017, + "chunks": 238, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 237, + "drafted": 25, + "accepted": 18, + "acceptance_rate": 0.72, + "tokens_per_round": 1.080168776371308 + } + }, + { + "sent_s": 0.0005138750000241998, + "first_s": 0.10387541700004022, + "last_s": 2.4977113749999944, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.000378334000004088, + "first_s": 0.08678854200002206, + "last_s": 2.4975427090000153, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + } + }, + { + "sent_s": 0.0, + "first_s": 0.06975508400000763, + "last_s": 2.4973653750000153, + "chunks": 234, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + } + } + ] + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 16, + "tps": 798.2396077591403, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.858300001042153e-05, + "first_s": 0.17979529199999433, + "last_s": 5.137077792000014, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0007607500000403888, + "first_s": 0.26507354200003874, + "last_s": 5.1373404170000185, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0001688749999857464, + "first_s": 0.14569525000001704, + "last_s": 5.13666766700004, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0005366249999951833, + "first_s": 0.09445470800000066, + "last_s": 5.137228125000036, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00022016700000904166, + "first_s": 0.24801995799998622, + "last_s": 5.13726516700001, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0005775000000198816, + "first_s": 0.16273537500001112, + "last_s": 5.136763833000032, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0006260420000216982, + "first_s": 0.06030212500002108, + "last_s": 5.136775083000032, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0002752920000261838, + "first_s": 0.11154883300002894, + "last_s": 5.137179000000003, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00047579200003156075, + "first_s": 0.04323062500003516, + "last_s": 5.136403292000011, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00032704200003763617, + "first_s": 0.026093166999999085, + "last_s": 5.136304792000033, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0008387080000034075, + "first_s": 0.2821312500000204, + "last_s": 5.137132000000008, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0003770000000145046, + "first_s": 0.1968186249999917, + "last_s": 5.13730370799999, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0008963750000248183, + "first_s": 0.0773855420000018, + "last_s": 5.136950667000008, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0006831670000337908, + "first_s": 0.2138946250000231, + "last_s": 5.1368389579999985, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.00042737500001521767, + "first_s": 0.12860412500003804, + "last_s": 5.136904917000038, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0, + "first_s": 0.23097549999999956, + "last_s": 5.137033458000019, + "chunks": 253, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + } + } + ] + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 1, + "tps": 356.16467232003214, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.020453832999976385, + "last_s": 0.7364147909999588, + "chunks": 92, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + } + } + ] + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 4, + "tps": 618.3910873475105, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.00010041600000931794, + "first_s": 0.027887832999965667, + "last_s": 1.6771637909999981, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00017733299995370544, + "first_s": 0.04744333299998971, + "last_s": 1.6771732079999992, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.00023887499997954365, + "first_s": 0.08631983300000456, + "last_s": 1.6773294579999742, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + } + }, + { + "sent_s": 0.0, + "first_s": 0.06689787499999511, + "last_s": 1.6771853329999544, + "chunks": 250, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + } + } + ] + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 8, + "tps": 796.428603323352, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.195899997394008e-05, + "first_s": 0.0462688339999886, + "last_s": 2.5843012919999637, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.000409708999995928, + "first_s": 0.16280487500000618, + "last_s": 2.5882531249999943, + "chunks": 256, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 255, + "drafted": 6, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + } + }, + { + "sent_s": 0.00016158400001131668, + "first_s": 0.10456841699999586, + "last_s": 2.5847895839999637, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0002703749999568572, + "first_s": 0.06574124999997366, + "last_s": 2.5843564999999558, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00021745899999814355, + "first_s": 0.08518424999999752, + "last_s": 2.584339166999996, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0005527089999759482, + "first_s": 0.14339274999997542, + "last_s": 2.584606416999975, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00034283399998003006, + "first_s": 0.12398116700001083, + "last_s": 2.584396708999975, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0, + "first_s": 0.026818249999962518, + "last_s": 2.5842485839999654, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + } + } + ] + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 16, + "tps": 791.8338766948201, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.87500000064756e-05, + "first_s": 0.22255808299996716, + "last_s": 5.180806458000006, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0007360000000176115, + "first_s": 0.18369995799997696, + "last_s": 5.180368541000007, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00022083300001440875, + "first_s": 0.12557158300000992, + "last_s": 5.180714582999997, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0006382909999729236, + "first_s": 0.2806921249999732, + "last_s": 5.180587749999972, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00016958300000169402, + "first_s": 0.028537165999978242, + "last_s": 5.180937416000006, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0008457910000174707, + "first_s": 0.14491249999997535, + "last_s": 5.180527457999972, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0005273329999795351, + "first_s": 0.0479936250000037, + "last_s": 5.179904207999982, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00027062499998464773, + "first_s": 0.20311637499997914, + "last_s": 5.180457082999965, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0003679579999698035, + "first_s": 0.16430674999998018, + "last_s": 5.180646958000011, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0005860409999627336, + "first_s": 0.10620045799998934, + "last_s": 5.179984499999989, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0007962909999719159, + "first_s": 0.3195373749999817, + "last_s": 5.181050749999997, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0006836249999651045, + "first_s": 0.30011095800000476, + "last_s": 5.181133082999963, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0004127500000095097, + "first_s": 0.261269249999998, + "last_s": 5.180873165999969, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00032120800000257077, + "first_s": 0.0674458329999652, + "last_s": 5.180086749999987, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.00045554100000799735, + "first_s": 0.24194262499997876, + "last_s": 5.181006707999984, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + }, + { + "sent_s": 0.0, + "first_s": 0.08683024999999134, + "last_s": 5.180760291000013, + "chunks": 255, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "tps": 289.11549612823666, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019241749999991953, + "last_s": 0.9012422079999851, + "chunks": 217, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "tps": 661.6906341663944, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 0.00010058300000537201, + "first_s": 0.018040291999966485, + "last_s": 1.5595010829999865, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 63, + "accepted": 37, + "acceptance_rate": 0.5873015873015873, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.00024054199997181058, + "first_s": 0.03435920800001213, + "last_s": 1.5595186670000203, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 63, + "accepted": 37, + "acceptance_rate": 0.5873015873015873, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.00018033300000297459, + "first_s": 0.06704200000001492, + "last_s": 1.5595461670000077, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + }, + { + "sent_s": 0.0, + "first_s": 0.05069608299999118, + "last_s": 1.5595314999999914, + "chunks": 219, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 218, + "drafted": 54, + "accepted": 37, + "acceptance_rate": 0.6851851851851852, + "tokens_per_round": 1.1743119266055047 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "tps": 851.92509905097, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.675000001152512e-05, + "first_s": 0.12320379200002662, + "last_s": 2.4198644169999852, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.0005418329999997695, + "first_s": 0.0742406670000264, + "last_s": 2.41535650000003, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00017433300001812313, + "first_s": 0.13952354200000627, + "last_s": 2.419875917000013, + "chunks": 225, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + } + }, + { + "sent_s": 0.00036108300002979377, + "first_s": 0.10689100000001872, + "last_s": 2.415414583000029, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00027833299998292205, + "first_s": 0.025299207999978535, + "last_s": 2.414964291999979, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0004121250000252985, + "first_s": 0.09056883300002028, + "last_s": 2.4152982080000243, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.00022708300002705073, + "first_s": 0.04163458300001821, + "last_s": 2.4150827500000105, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + }, + { + "sent_s": 0.0, + "first_s": 0.057957625000028656, + "last_s": 2.4149834579999947, + "chunks": 224, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "tps": 849.7539917935715, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "requests": [ + { + "sent_s": 9.229200003346705e-05, + "first_s": 0.1563097500000481, + "last_s": 4.817629250000039, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0007655420000105551, + "first_s": 0.09106545800000276, + "last_s": 4.818126333000009, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0005632080000168571, + "first_s": 0.025629167000033704, + "last_s": 4.817198333000022, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00021450000002687375, + "first_s": 0.20527954200002796, + "last_s": 4.817866583000011, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0001658750000501641, + "first_s": 0.05837937500001544, + "last_s": 4.817788542000017, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0009786670000266895, + "first_s": 0.2706689580000443, + "last_s": 4.827018792000047, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.0006769170000211489, + "first_s": 0.12370716700002049, + "last_s": 4.81748595800002, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00027220800001259704, + "first_s": 0.22165316699999948, + "last_s": 4.817694750000044, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0003266670000243721, + "first_s": 0.2543046670000422, + "last_s": 4.827006958000027, + "chunks": 237, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 236, + "drafted": 49, + "accepted": 19, + "acceptance_rate": 0.3877551020408163, + "tokens_per_round": 1.0847457627118644 + } + }, + { + "sent_s": 0.0007246250000321197, + "first_s": 0.1889802920000534, + "last_s": 4.818011333000015, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0006133750000003602, + "first_s": 0.04200358300005291, + "last_s": 4.817208667000045, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00037904200002003563, + "first_s": 0.13998337500004254, + "last_s": 4.817958458000021, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.00047962500002540764, + "first_s": 0.0747155829999997, + "last_s": 4.818229042000041, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0011613750000378786, + "first_s": 0.2379564170000208, + "last_s": 4.818185667000023, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0004290420000074846, + "first_s": 0.17260258300001396, + "last_s": 4.817737625000007, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + }, + { + "sent_s": 0.0, + "first_s": 0.10742200000004232, + "last_s": 4.817548625000029, + "chunks": 235, + "prompt_tokens": 14, + "completion_tokens": 256, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 1, + "tps": 261.22060964024485, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 0.0, + "first_s": 0.019794375000003583, + "last_s": 0.9959807500000011, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 4, + "tps": 609.6870758668199, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.979099996826335e-05, + "first_s": 0.027631165999991936, + "last_s": 1.7005795000000035, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 35, + "accepted": 8, + "acceptance_rate": 0.22857142857142856, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.0002387909999583826, + "first_s": 0.08411141599998473, + "last_s": 1.700620540999978, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.00017691599998670426, + "first_s": 0.046487874999968426, + "last_s": 1.7005932499999972, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 35, + "accepted": 8, + "acceptance_rate": 0.22857142857142856, + "tokens_per_round": 1.0364372469635628 + } + }, + { + "sent_s": 0.0, + "first_s": 0.06533966599999985, + "last_s": 1.7006090409999501, + "chunks": 248, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 247, + "drafted": 30, + "accepted": 8, + "acceptance_rate": 0.26666666666666666, + "tokens_per_round": 1.0364372469635628 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 8, + "tps": 809.9225170234652, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.008300003188197e-05, + "first_s": 0.02065962500000751, + "last_s": 2.538832582999987, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00031466600000840117, + "first_s": 0.0401318330000322, + "last_s": 2.538852875000032, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00016016600000057224, + "first_s": 0.05894499999999425, + "last_s": 2.5388885830000163, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00037554099998260426, + "first_s": 0.07783116600000994, + "last_s": 2.539023708000002, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0004380000000310247, + "first_s": 0.11547354100002849, + "last_s": 2.5393518750000226, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.00020958300001439056, + "first_s": 0.1531026250000309, + "last_s": 2.539231583000003, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0002641660000222146, + "first_s": 0.09667970800001058, + "last_s": 2.538900791000003, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + } + }, + { + "sent_s": 0.0, + "first_s": 0.13430062500003714, + "last_s": 2.5394190829999843, + "chunks": 249, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + } + } + ] + }, + { + "round": 3, + "arm": "candidate-plain", + "prompt": "prose", + "streams": 16, + "tps": 802.5799140637832, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "requests": [ + { + "sent_s": 9.554199999683988e-05, + "first_s": 0.10360991700002842, + "last_s": 5.107608249999998, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00016604199998937474, + "first_s": 0.14120920800002068, + "last_s": 5.107471332999978, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005204170000183694, + "first_s": 0.17878975000002129, + "last_s": 5.107176707999997, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007186670000010054, + "first_s": 0.21637195800002473, + "last_s": 5.107288041999993, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00021729200000208948, + "first_s": 0.12241758300001493, + "last_s": 5.107812749999994, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00032437499999105057, + "first_s": 0.25404087499998695, + "last_s": 5.10714120800003, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.00027066699999522825, + "first_s": 0.028251749999981257, + "last_s": 5.106763291999982, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0004718750000165528, + "first_s": 0.27287325000003193, + "last_s": 5.107762917000002, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0008905419999791775, + "first_s": 0.2352070829999775, + "last_s": 5.107568833000016, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0007765419999827827, + "first_s": 0.19758012500000177, + "last_s": 5.107524416999979, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0005857080000168935, + "first_s": 0.31045379200003254, + "last_s": 5.111857666999981, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0003726670000219201, + "first_s": 0.06591500000001815, + "last_s": 5.106777333000025, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.000637874999995347, + "first_s": 0.08476883299999827, + "last_s": 5.107337874999985, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0004238750000240543, + "first_s": 0.16000133300002517, + "last_s": 5.107853707999993, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + }, + { + "sent_s": 0.0009781250000173713, + "first_s": 0.29165512499997703, + "last_s": 5.111846125, + "chunks": 253, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 252, + "drafted": 35, + "accepted": 3, + "acceptance_rate": 0.08571428571428572, + "tokens_per_round": 1.0158730158730158 + } + }, + { + "sent_s": 0.0, + "first_s": 0.04710833299998285, + "last_s": 5.106851082999981, + "chunks": 252, + "prompt_tokens": 18, + "completion_tokens": 256, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + } + } + ] + } + ], + "medians": [ + { + "prompt": "code", + "streams": 1, + "tps": { + "main": 289.3127543896282, + "published": 354.63683817375284, + "candidate-plain": 288.9588092703433, + "candidate-drafted": 356.3379766224038 + }, + "candidate_drafted_vs_published_pct": 0.47968464229808383, + "candidate_drafted_vs_main_pct": 23.167047154274535, + "candidate_plain_vs_main_pct": -0.12233996390226265 + }, + { + "prompt": "code", + "streams": 4, + "tps": { + "main": 652.2735791542232, + "published": 645.4865332378143, + "candidate-plain": 661.7877417761589, + "candidate-drafted": 633.0452266294983 + }, + "candidate_drafted_vs_published_pct": -1.9274308552820418, + "candidate_drafted_vs_main_pct": -2.94789688548438, + "candidate_plain_vs_main_pct": 1.4586153610993025 + }, + { + "prompt": "code", + "streams": 8, + "tps": { + "main": 830.8723123556888, + "published": 812.6499883768286, + "candidate-plain": 852.0692879199623, + "candidate-drafted": 812.5992670903888 + }, + "candidate_drafted_vs_published_pct": -0.00624146768783973, + "candidate_drafted_vs_main_pct": -2.199260342842846, + "candidate_plain_vs_main_pct": 2.5511712508719864 + }, + { + "prompt": "code", + "streams": 16, + "tps": { + "main": 830.128290576471, + "published": 798.8767760220219, + "candidate-plain": 849.7420828208678, + "candidate-drafted": 886.4564675049711 + }, + "candidate_drafted_vs_published_pct": 10.962853610421508, + "candidate_drafted_vs_main_pct": 6.785478529997291, + "candidate_plain_vs_main_pct": 2.3627422974316747 + }, + { + "prompt": "prose", + "streams": 1, + "tps": { + "main": 262.30748385704317, + "published": 356.77452074457415, + "candidate-plain": 261.3022597821349, + "candidate-drafted": 356.9475919964822 + }, + "candidate_drafted_vs_published_pct": 0.04850998091086023, + "candidate_drafted_vs_main_pct": 36.07983529399321, + "candidate_plain_vs_main_pct": -0.38322355890391124 + }, + { + "prompt": "prose", + "streams": 4, + "tps": { + "main": 622.7876024155925, + "published": 618.2932356793516, + "candidate-plain": 610.0894226933044, + "candidate-drafted": 615.7268551631641 + }, + "candidate_drafted_vs_published_pct": -0.41507497868186194, + "candidate_drafted_vs_main_pct": -1.1337327886813964, + "candidate_plain_vs_main_pct": -2.038926220277326 + }, + { + "prompt": "prose", + "streams": 8, + "tps": { + "main": 804.4487723340825, + "published": 792.2780785659455, + "candidate-plain": 810.0205684276797, + "candidate-drafted": 796.214425867501 + }, + "candidate_drafted_vs_published_pct": 0.49683910334619164, + "candidate_drafted_vs_main_pct": -1.0236010980152144, + "candidate_plain_vs_main_pct": 0.6926228599281492 + }, + { + "prompt": "prose", + "streams": 16, + "tps": { + "main": 798.6060418170423, + "published": 792.6643643406294, + "candidate-plain": 802.7420492371732, + "candidate-drafted": 862.9112022985937 + }, + "candidate_drafted_vs_published_pct": 8.862116315320744, + "candidate_drafted_vs_main_pct": 8.052175555201146, + "candidate_plain_vs_main_pct": 0.5179033470270822 + } + ], + "m5_qualification": { + "operator_summary": "SUMMARY 11 kernels: 11 pass gates 1-4, 0 fail, 0 known findings; 17 controls: 17 red as planted, 0 missed; 0 limit problems; gates 53558 ms, limits 110 ms", + "drafter_reference": { + "picks_equal": 9, + "picks": 9, + "worst_confidence_difference": 0.00158, + "steady_rounds": 20, + "host_allocations": 0, + "resizes": 0 + }, + "stress": [ + { + "slots": 6, + "seconds": 60, + "completed": 423, + "cancelled": 233, + "wrong": 0, + "errors": [] + }, + { + "slots": 16, + "seconds": 90, + "completed": 845, + "cancelled": 455, + "wrong": 0, + "errors": [] + } + ], + "sampled": [ + { + "streams": 1, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ] + ] + }, + { + "streams": 4, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ] + ] + }, + { + "streams": 8, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ] + ] + }, + { + "streams": 16, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ], + [ + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986" + ], + [ + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351" + ], + [ + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369" + ], + [ + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a" + ], + [ + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363" + ], + [ + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398" + ], + [ + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db" + ], + [ + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd" + ] + ] + } + ], + "resume": { + "equal": true, + "cached_tokens": 3517, + "resumed_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91", + "fresh_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91" + }, + "8bit": [ + { + "streams": 1, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 4, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 8, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 16, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 1, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 4, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 8, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 16, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + } + ], + "long_context": [ + { + "length": 65536, + "plain_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "solo_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "concurrent_sha256": [ + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6" + ] + }, + { + "length": 130816, + "plain_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "solo_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "concurrent_sha256": [ + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3" + ] + } + ], + "shutdowns": { + "tokens.shutdown": { + "forced_kill": false, + "seconds": 0.5550237089999999, + "returncode": 0 + }, + "final-1-candidate-drafted.shutdown": { + "forced_kill": false, + "seconds": 0.44267137500003173, + "returncode": 0 + }, + "final-0-published.shutdown": { + "forced_kill": false, + "seconds": 0.3199985830000003, + "returncode": 0 + }, + "final-2-candidate-plain.shutdown": { + "forced_kill": false, + "seconds": 0.3152450830000362, + "returncode": 0 + }, + "final-1-main.shutdown": { + "forced_kill": false, + "seconds": 0.26646854199998415, + "returncode": 0 + }, + "final-2-candidate-drafted.shutdown": { + "forced_kill": false, + "seconds": 0.7430550000000267, + "returncode": 0 + }, + "final-3-published.shutdown": { + "forced_kill": false, + "seconds": 0.7333516659999759, + "returncode": 0 + }, + "final-3-candidate-plain.shutdown": { + "forced_kill": false, + "seconds": 0.3819492080000373, + "returncode": 0 + }, + "final-2-main.shutdown": { + "forced_kill": false, + "seconds": 0.4357252500000186, + "returncode": 0 + }, + "functional-fresh.shutdown": { + "forced_kill": false, + "seconds": 0.551300375000011, + "returncode": 0 + }, + "final-0-candidate-plain.shutdown": { + "forced_kill": false, + "seconds": 0.49287025000000995, + "returncode": 0 + }, + "functional-cache.shutdown": { + "forced_kill": false, + "seconds": 0.5678107079999961, + "returncode": 0 + }, + "final-3-main.shutdown": { + "forced_kill": false, + "seconds": 0.31679716700000426, + "returncode": 0 + }, + "final-0-main.shutdown": { + "forced_kill": false, + "seconds": 0.3809736660000027, + "returncode": 0 + }, + "8bit-equality.shutdown": { + "forced_kill": false, + "seconds": 0.48993237499996667, + "returncode": 0 + }, + "final-1-published.shutdown": { + "forced_kill": false, + "seconds": 0.7285161659999915, + "returncode": 0 + }, + "long-serving.shutdown": { + "forced_kill": false, + "seconds": 0.7801908750000166, + "returncode": 0 + }, + "final-3-candidate-drafted.shutdown": { + "forced_kill": false, + "seconds": 0.5634543329999815, + "returncode": 0 + }, + "final-2-published.shutdown": { + "forced_kill": false, + "seconds": 0.44117920899998353, + "returncode": 0 + }, + "final-1-candidate-plain.shutdown": { + "forced_kill": false, + "seconds": 0.48965824999999086, + "returncode": 0 + }, + "final-0-candidate-drafted.shutdown": { + "forced_kill": false, + "seconds": 0.48779195900002037, + "returncode": 0 + } + } + }, + "host_checks": { + "focused": { + "steps_passed": 69, + "steps_total": 69, + "tests_passed": 117, + "tests_total": 117 + }, + "default_build": { + "steps_passed": 239, + "steps_total": 239 + }, + "stateful_proposer_regression": "passed", + "granted_depth_tests": 14, + "copy_tests": 34, + "lean": { + "main_findings": 425, + "candidate_findings": 425, + "new_findings": 0 + } + }, + "m4_qualification": { + "device": "Apple M4 Pro, 20 GPU cores, 48 GiB, macOS 26.5.1", + "operator_summary": "SUMMARY 11 kernels: 11 pass gates 1-4, 0 fail, 0 known findings; 17 controls: 17 red as planted, 0 missed; 0 limit problems; gates 66369 ms, limits 192 ms", + "drafter_reference": { + "picks_equal": 9, + "picks": 9, + "worst_confidence_difference": 0.00106, + "steady_rounds": 20, + "host_allocations": 0, + "resizes": 0 + }, + "serving_status": "stopped at the resumed-cache prerequisite: OS memory pressure explicitly disabled keeping prompt states; cached_tokens=0; stress/sample/8bit/memory phases did not run", + "pressure_warning": "one more stream only when none runs, no new kept prompt states", + "shutdown": { + "forced_kill": false, + "seconds": 0.3489787499420345, + "returncode": 0 + }, + "performance_status": "not qualified under the observed CPU/memory contention" + }, + "load_memory": { + "device": "M5 Ultra", + "context": 4096, + "parallel": 1, + "before_generation": true, + "arms": { + "published": { + "context_window": 4096, + "context_fitted": false, + "process": { + "source": "darwin_rusage_info_v4", + "physical_footprint_bytes": 15696735176, + "lifetime_peak_physical_footprint_bytes": 15709645744 + } + }, + "candidate": { + "context_window": 4096, + "context_fitted": false, + "process": { + "source": "darwin_rusage_info_v4", + "physical_footprint_bytes": 15679974464, + "lifetime_peak_physical_footprint_bytes": 15733451768 + } + } + }, + "candidate_peak_delta_bytes": 23806024 + }, + "limitations": [ + "A material four-stream code regression remains; the candidate is not a universal throughput win.", + "Candidate-plain four-stream prose also regresses versus main.", + "The additional shared calibration increases hot-model startup time by roughly one second in this run.", + "M4 served cache/concurrency qualification could not complete under system memory pressure; it is not recorded as passed.", + "M1/M2/M3/CUDA/HIP were not rerun; focused LFM2 operators are not the full cross-family kernel-contract gate." + ], + "load_readiness": [ + { + "event": "ready", + "label": "final-0-main", + "seconds": 2.5573577919999906 + }, + { + "event": "ready", + "label": "final-0-published", + "seconds": 2.5535984580000104 + }, + { + "event": "ready", + "label": "final-0-candidate-plain", + "seconds": 3.573031125 + }, + { + "event": "ready", + "label": "final-0-candidate-drafted", + "seconds": 3.572747667000016 + }, + { + "event": "ready", + "label": "final-1-published", + "seconds": 2.5532073749999995 + }, + { + "event": "ready", + "label": "final-1-candidate-plain", + "seconds": 3.0623082079999904 + }, + { + "event": "ready", + "label": "final-1-candidate-drafted", + "seconds": 3.564061958999986 + }, + { + "event": "ready", + "label": "final-1-main", + "seconds": 1.5315142910000077 + }, + { + "event": "ready", + "label": "final-2-candidate-plain", + "seconds": 3.063439207999977 + }, + { + "event": "ready", + "label": "final-2-candidate-drafted", + "seconds": 3.5726659590000054 + }, + { + "event": "ready", + "label": "final-2-main", + "seconds": 2.0429697499999975 + }, + { + "event": "ready", + "label": "final-2-published", + "seconds": 2.552820082999972 + }, + { + "event": "ready", + "label": "final-3-candidate-drafted", + "seconds": 3.5727163330000167 + }, + { + "event": "ready", + "label": "final-3-main", + "seconds": 2.042876958000022 + }, + { + "event": "ready", + "label": "final-3-published", + "seconds": 2.5485449579999795 + }, + { + "event": "ready", + "label": "final-3-candidate-plain", + "seconds": 3.054198165999992 + } + ], + "review_fixes": { + "rejected_guard_commits_excluded_from_ancestry": [ + "5d9fcf68f87dd46ff653c17a2fd937fd09074ce1", + "1b57e583855b552aa003004e80db702ac1e6ae4e", + "4fa02928a7484ccf7bc1a529936618e9489eb24e", + "b66a7629e60d7283d5a837f7b3d2632f03000896", + "6539b72beacf5bf911d5a5141f50da88cfcab4c6" + ] + } +} diff --git a/docs/receipts/lfm2-dspark-review/wide_matrix.py b/docs/receipts/lfm2-dspark-review/wide_matrix.py new file mode 100644 index 0000000000..7f672c2ea8 --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/wide_matrix.py @@ -0,0 +1,30 @@ +# Four-arm Metal serving comparison; reserve the GPU exclusively before running. +import importlib.util,os,fcntl,threading,concurrent.futures as cf,hashlib +from pathlib import Path +s=importlib.util.spec_from_file_location('q',Path(__file__).with_name('qualification.py'));q=importlib.util.module_from_spec(s);s.loader.exec_module(q) +root=Path(os.environ.get('TF_QUAL_CANDIDATE_ROOT',str(q.ROOT)));q.OUT=Path(os.environ.get('TF_QUAL_OUT','/tmp/tf-lfm2-wide-matrix'));q.OUT.mkdir(exist_ok=True) +with open('/tmp/tensorfold-model.lock','a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX) + q.save('source-hashes',{p:hashlib.sha256((root/p).read_bytes()).hexdigest() for p in ['zig/src/core/lanes/depth.zig','zig/src/core/lanes/drafted_chances_test.zig','zig/src/families/lfm2_moe/backend.zig','zig/src/families/lfm2_moe/dspark.zig','zig/src/families/lfm2_moe/dspark_block.zig']}) + arms={'main':(Path(os.environ['TF_QUAL_MAIN']),'plain'),'published':(Path(os.environ['TF_QUAL_PUBLISHED']),'bf16'),'candidate-plain':(q.CAND,'plain'),'candidate-drafted':(q.CAND,'bf16')} + q.save('binaries',{name:hashlib.sha256(binary.read_bytes()).hexdigest() for name,(binary,_) in arms.items()}) + results=[];refs={} + prompts=['Write a short Python function that computes the Fibonacci sequence and explain it.','Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.'] + for round_ in range(4): + order=list(arms);offset=round_%len(order);order=order[offset:]+order[:offset] + for arm in order: + binary,mode=arms[arm] + label=f'final-{round_}-{arm}' + with q.Server(label,binary,mode,4096,16): + if 'memory pressure:' in (q.OUT/(label+'.server.log')).read_text(errors='replace'):raise RuntimeError('memory pressure prevents a valid concurrency qualification') + for prompt in prompts: + q.post({'prompt':prompt,'max_tokens':32}) + for width in [1,4,8,16]: + gate=threading.Barrier(width) + def run(i):gate.wait();return q.post({'prompt':prompt,'max_tokens':256},True) + with cf.ThreadPoolExecutor(width) as ex:rows=list(ex.map(run,range(width))) + ref=refs.setdefault(prompt,q.digest(rows[0]));assert all(q.digest(r)==ref for r in rows) + if 'memory pressure:' in (q.OUT/(label+'.server.log')).read_text(errors='replace'):raise RuntimeError('memory pressure changed stream admission during qualification') + rate=width*255/(max(r['last'] for r in rows)-min(r['first'] for r in rows)) + item={'round':round_,'arm':arm,'prompt':'code' if prompt.startswith('Write') else 'prose','streams':width,'tps':rate,'requests':rows};results.append(item);q.save('comparison',results);q.emit('comparison',**{k:v for k,v in item.items() if k!='requests'}) + q.emit('comparison-done',cells=len(results)) diff --git a/docs/recipes/README.md b/docs/recipes/README.md index 8017124388..0cfdf94f1b 100644 --- a/docs/recipes/README.md +++ b/docs/recipes/README.md @@ -22,7 +22,7 @@ For agent-assisted contributions, read the [TensorFold contributor skill](SKILL. | Qwen3.6-35B-A3B | [MLX with MTP drafts, one-GPU CUDA](qwen3.6-moe.md) | | Step-5-Preview | [Native Metal on one 256 GB Mac: 2-bit experts, MTP drafts, images](step-5-preview.md) | | Kolibri-1 | [Native CUDA, block FP8, research checkpoint](kolibri-1.md) | -| LFM2.5-24B-A2B | [Native Metal, MLX 4-bit, shared rounds](lfm2.5-24b-a2b.md) | +| LFM2.5-24B-A2B | [Native Metal, MLX 4-bit, shared rounds, DSpark drafts](lfm2.5-24b-a2b.md) | | Spark-X2.6-4B | [Native CUDA](spark-x2.6-4b-cuda.md), [native Metal, the bf16 release, contexts to 1M](spark-x2.6-4b-metal.md) | ## Capability floor diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index 7b8b2cdf4b..a3fce056e5 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -41,3 +41,84 @@ the shared-round kernels on it. The M4 Pro is not yet measured on a quiet machin Every reply was token for token equal before and after, and equal to its solo run and to `"draft": false`. Cold prefill at 2k, 8k and 32k tokens and peak memory did not change. + +## DSpark drafts + +LiquidAI's DSpark drafter for this model is +[`LiquidAI/LFM2.5-24B-A2B-DSpark`](https://huggingface.co/LiquidAI/LFM2.5-24B-A2B-DSpark/tree/e052e5b1dd103c38cdd4f8ddc068d7dea4586f55) +at revision `e052e5b1dd103c38cdd4f8ddc068d7dea4586f55` (327.7M bf16 parameters, 655 MB). Its weights are under the +[LFM Open License v1.0](https://huggingface.co/LiquidAI/LFM2.5-24B-A2B-DSpark/blob/e052e5b1dd103c38cdd4f8ddc068d7dea4586f55/LICENSE); +TensorFold does not ship them, so read the license before use. + +```bash +hf download LiquidAI/LFM2.5-24B-A2B-DSpark --revision e052e5b1dd103c38cdd4f8ddc068d7dea4586f55 \ + config.json model.safetensors LICENSE README.md --local-dir "$HOME/models/LFM2.5-24B-A2B-DSpark" +zig-out/native/bin/tensorfold-native serve "$MODEL_DIR" --port 8080 --parallel 16 \ + --drafter "$HOME/models/LFM2.5-24B-A2B-DSpark" +``` + +Without `--drafter` the engine serves as before. `"draft": false` on a request, or `--no-drafts`, turns the drafter +off; the lane core verifies every draft, so a drafted reply equals its `"draft": false` reply token for token. + +The drafter has no embedding and no head of its own: + +- Context: the target's rows after layers 2, 10, 18, 26 and 34 (the backend's taps, `lfm2_tap`), through `fc` and + `hidden_norm`, give each of its five layers' context keys and values (k_norm, then RoPE at the row's position). +- Block: the pending token and mask tokens run five Qwen3 layers with non-causal attention over the context and the + block, then the target's head. A block is as long as the deepest chain its pass holds, at most the nine rows the + drafter was trained on. +- Markov head: row i's logits plus `W2 W1[previous token]` (rank 256); draft i is their argmax. + +The drafter runs the released bf16 matrices as they are (`--drafter-bits 0`, the default). `--drafter-bits 8` instead +quantizes them at load to 8-bit MLX affine codes in groups of 64, about 5% faster on one stream and 361 MiB of weights +against 625 MiB; the server logs which mode runs, and 4 bits is refused. Both run on the family's kernels +(bf16 on `lfm2_bf16_rows`, `lfm2_bf16_mma` and `lfm2_markov_scan_bf16`, 8-bit on `core/tile_qmm`); the q and k rows are reordered so that the target's half-split RoPE kernel gives the +checkpoint's interleaved RoPE. Its own kernels are `lfm2_tap`, `lfm2_attn_block` and `lfm2_attn_block_merge` +(non-causal block attention, the target's attention kernels with a template flag), `lfm2_markov_scan` and +`lfm2_markov_pick` (a Markov step) and `lfm2_draft_ids` (a hold's drafts into the next verify's rows on the GPU, so +the host does not wait for a hold that fits one pass). Each kept prompt state keeps the drafter's context too, so a +drafting stream resumes prompts. The lane core prices drafts across the admitted stream counts. + +`tf-kernel-contract --only lfm2` checks these kernels. `tf-lfm2-dspark MODEL DRAFTER TOKENS [ROWS]` runs the drafter +against an fp32 host reference of the released checkpoint (the same taps and the target's embedding and head), +prints each draft against the reference's pick with the margin between the reference's two best tokens, and times +the drafter's passes; a near tie can pick the other token. + +### Verify schedule + +The drafter also carries a confidence head: a linear map of a draft's final block row and `W1[previous token]` +(2,304 inputs) to one logit, through a sigmoid (`lfm2_dspark_confidence`). It does not change the drafts. +The verify rows follow it: a chain's chance at draft j is the product of its first j confidences. Each round the +lane core's window allocator takes the likeliest drafts over all streams while the expected tokens a ms rise, so +each stream gets its own verify length, and the next hold drafts as deep as the stream's grant. A hold still +running on the GPU is priced by its lane's last confidences, so the host does not wait for it. A checkpoint +without the head verifies by the core's depth rule, from each stream's measured acceptance by depth. + +A stream with a copy ahead verifies the copy and asks no hold, and a drafting stream prices its copy +windows by its own copies. The lane core also weighs the hold pass itself: the drafter times holds of 1 to 16 +blocks at every admissible depth at load and corrects those prices toward each settled hold's GPU time. +Prices interpolate neighboring measured depths, preserving the cost changes at kernel transitions. + +`tf-lfm2-dspark` also prints the confidence +head against its fp32 reference and times shared holds of 2 to 16 streams. + +The loader reads the checkpoint one matrix at a time and keeps the released bf16 matrices by default, or the +8-bit codes with `--drafter-bits 8`; a stream's drafter lane is one +Metal buffer (its layers' keys and values, then its staged rows), and its bytes join the streams the load limit +admits. At load the drafter logs its weights, scratch and a stream's bytes. `tf-lfm2-dspark` also counts host +allocations over twenty served drafter rounds, which must be none. + +A hold runs on one concurrent encoder with a barrier only where a step reads an earlier step's writes: q, k and v, +and gate and up, share one sums pass, and the blocks of a shared hold attend side by side. An 8-bit projection takes +the scalar twin up to 8 rows, the window kernel past them where the GPU has no tensor units, and the tensor units +from 9 rows (the head from 5) where it has them. Held blocks use one bf16 16-row group, then the existing +64x64 tiles past 16 rows, avoiding a second weight pass for shared holds of 17 to 32 rows. Context ingestion +keeps the original row kernel through 32 rows, preserving the persistent context projections. +When one lane's stage of 128 verified rows fills, its absorb pass takes +every lane's staged rows, so the lanes' stages fill together. + +Shared verification past the 16-row qualified window uses measured increments +from the shared forward path through 32 rows, anchored at the 16-row window +price. This calibrates the batch cost without widening a stream's qualified +window. The wide-batch receipt records the throughput gains, the accepted +four-stream tradeoff, startup/footprint cost and hardware coverage limits. diff --git a/zig/build/metal_targets.zig b/zig/build/metal_targets.zig index 8a16d7432e..df5a809e3f 100644 --- a/zig/build/metal_targets.zig +++ b/zig/build/metal_targets.zig @@ -294,6 +294,7 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s .{ .name = "tf-lfm2-forward", .path = "zig/tests/lfm2_forward.zig", .about = "LFM2-MoE native teacher-forced logits" }, .{ .name = "tf-lfm2-bench", .path = "zig/tests/lfm2_bench.zig", .about = "LFM2-MoE prompt-chunk and one-row decode timing" }, .{ .name = "tf-lfm2-exact", .path = "zig/tests/lfm2_exact.zig", .about = "LFM2-MoE prompt chunks, windows and committed-state exactness" }, + .{ .name = "tf-lfm2-dspark", .path = "zig/tests/lfm2_dspark.zig", .about = "LFM2's DSpark drafter on the GPU against an fp32 host reference of the released drafter" }, .{ .name = "tf-lfm2-fp64", .path = "zig/tests/lfm2_fp64.zig", .about = "LFM2 projection classes and attention on the tensor units, the tiles and the twin, each against float64" }, .{ .name = "tensorfold", .path = "zig/src/main.zig", .about = "The native engine's command line" }, .{ .name = "tf-nemotron-fixtures", .path = "zig/tests/nemotron_fixtures.zig", .about = "Nemotron kernels against the Python engine's captured ops" }, @@ -462,6 +463,8 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s b.step("test-nemotron-round-depth-host", "Priced head depth, nonlanding levels and fixed-depth controls without devices").dependOn(&round_depth.step); const lfm2_tests = b.addRunArtifact(b.addTest(.{ .root_module = engine, .filters = &.{"lfm2_moe"} })); b.step("test-lfm2", "The LFM2-MoE family's tests: config, draws, state and its shared rounds on this Mac's GPU").dependOn(&lfm2_tests.step); + const lfm2_dspark = b.addRunArtifact(b.addTest(.{ .root_module = engine, .filters = &.{"lfm2 dspark:"} })); + b.step("test-lfm2-dspark", "LFM2's DSpark drafter and its taps: config, quantizer and kernels on this Mac's GPU").dependOn(&lfm2_dspark.step); test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = lanes })).step); // the fake fabric maps its windows with mmap: the Macs' RDMA, not Windows if (target.result.os.tag != .windows) test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = fabric })).step); diff --git a/zig/kernels/metal/lfm2/lfm2.metal b/zig/kernels/metal/lfm2/lfm2.metal index af6a2a7372..2da8578970 100644 --- a/zig/kernels/metal/lfm2/lfm2.metal +++ b/zig/kernels/metal/lfm2/lfm2.metal @@ -232,7 +232,7 @@ inline void lfm2_stage(const thread uint4* pre, threadgroup uint* ks, threadgrou } // Every row's attention partials (one kernel for prompt and decode): a threadgroup a (chunk, kv head, 8 * RB rows). -template // a simdgroup's RB row blocks share each staged K and V fragment, each block's arithmetic its own +template // RB row blocks share each staged K and V fragment; FULL: all see [0, len + 1) inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* keys, const device bfloat* values, lfm2_attn_args p, device float* pm, device float* pl, device float* po, uint rows, uint3 tg, uint tid, uint sg, uint lane, threadgroup uint (*ks)[HD * (KT / 2)], @@ -242,8 +242,8 @@ inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* ke const uint kh = tg.y, r0 = tg.x * 8 * RB, head = kh * (QH / KVH) + sg; const uint qid = lane / 4, fm = (qid & 4) + ((lane / 2) % 4), fn = (qid & 2) * 2 + (lane % 2) * 2; uint count[RB]; // this lane's rows' visible keys - for (int b = 0; b < RB; ++b) count[b] = p.len + min(r0 + 8 * b + fm, rows - 1) + 1; - const uint top = p.len + min(r0 + 8 * RB - 1, rows - 1) + 1; // the tile's most + for (int b = 0; b < RB; ++b) count[b] = FULL ? p.len + 1 : p.len + min(r0 + 8 * b + fm, rows - 1) + 1; + const uint top = FULL ? p.len + 1 : p.len + min(r0 + 8 * RB - 1, rows - 1) + 1; // the tile's most if (chunk * PER + sub0 * SUB >= top) return; vec q[RB][8]; for (int b = 0; b < RB; ++b) @@ -340,7 +340,7 @@ inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* ke for (int d = 0; d < 8; ++d) *(device float2*)(po + at * HD + 8 * d + fn) = O[b][d]; } } -template +template [[kernel]] void lfm2_attn_mma(const device bfloat* queries [[buffer(0)]], const device bfloat* keys [[buffer(1)]], const device bfloat* values [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]], device float* pm [[buffer(4)]], device float* pl [[buffer(5)]], device float* po [[buffer(6)]], @@ -349,15 +349,16 @@ template uint lane [[thread_index_in_simdgroup]]) { threadgroup uint ks[2][HD * (KT / 2)]; threadgroup bfloat vs[2][KB * KP]; - lfm2_attn_rows(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs); + lfm2_attn_rows(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs); } template [[host_name("lfm2_attn_mma")]] [[kernel]] decltype(lfm2_attn_mma<1>) lfm2_attn_mma<1>; template [[host_name("lfm2_attn_mma_r2")]] [[kernel]] decltype(lfm2_attn_mma<2>) lfm2_attn_mma<2>; // The chunks merged in chunk order for (head, row): Y [rows, QH*HD] bf16. +template inline void lfm2_merge_row(const device float* pm, const device float* pl, const device float* po, lfm2_attn_args p, device bfloat* y, uint head, uint r, uint lane) { - const uint count = p.len + r + 1; + const uint count = FULL ? p.len + 1 : p.len + r + 1; const uint per = PER; const uint chunks = (count + per - 1) / per; const size_t at = (size_t(r) * QH + head) * (CHUNKS * p.subs); @@ -391,12 +392,15 @@ inline void lfm2_merge_row(const device float* pm, const device float* pl, const out[lane] = static_cast(o0 / l); out[lane + 32] = static_cast(o1 / l); } +template [[kernel]] void lfm2_attn_merge(const device float* pm [[buffer(0)]], const device float* pl [[buffer(1)]], const device float* po [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]], device bfloat* y [[buffer(4)]], uint2 tg [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) { - lfm2_merge_row(pm, pl, po, p, y, tg.x, tg.y, lane); + lfm2_merge_row(pm, pl, po, p, y, tg.x, tg.y, lane); } +template [[host_name("lfm2_attn_merge")]] [[kernel]] decltype(lfm2_attn_merge) lfm2_attn_merge; +template [[host_name("lfm2_attn_block_merge")]] [[kernel]] decltype(lfm2_attn_merge) lfm2_attn_merge; // Router logits [rows, E] in fp32: a simdgroup an expert, lane l over inputs 4l..4l+3 of each 128, then simd_sum. [[kernel]] void lfm2_route_logits(const device bfloat* x [[buffer(0)]], const device float* router [[buffer(1)]], @@ -568,3 +572,11 @@ constant constexpr uint SORT_CHUNKS = 16; // thread (e, c) counts expert e's pi best[2 * row + 1] = bad; } } + +// A drafter's tap: H's rows [rows, D] copied into TAPS at column `at.y` of rows `at.x` values apart, 8 a thread. +[[kernel]] void lfm2_tap(const device bfloat* h [[buffer(0)]], device bfloat* taps [[buffer(1)]], constant uint2& at [[buffer(2)]], + uint2 i [[thread_position_in_grid]]) { + if (i.x >= D / 8) return; + const device vec* from = (const device vec*)(h + size_t(i.y) * D); + ((device vec*)(taps + size_t(i.y) * at.x + at.y))[i.x] = from[i.x]; +} diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal new file mode 100644 index 0000000000..7440805007 --- /dev/null +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -0,0 +1,258 @@ +// DSpark's Markov step, compiled after lfm2.metal's text (its includes and constants). + +// DSpark's Markov scan, W2 as 8-bit codes (BF16 false) or bf16 as released (true): a threadgroup's best of 512 tokens. +struct lfm2_markov_args { uint stride, rank, vocab, prev_stride; }; +constant constexpr uint MARKOV_SPAN = 512; + +// Eight bf16 in a uint4 as floats (a bf16 is the high half of its float). +inline void lfm2_unpack8(uint4 v, thread float* f) { + for (int j = 0; j < 4; ++j) { + f[2 * j] = as_type(v[j] << 16); + f[2 * j + 1] = as_type(v[j] & 0xffff0000u); + } +} +template +inline void lfm2_markov_scan_t(const device bfloat* logits, const device bfloat* w1, const device uint8_t* w2, + const device uint* prev, constant lfm2_markov_args& p, device float* best, + device uint* at, const device bfloat* w2s, const device bfloat* w2b, uint2 tg, + uint2 grid, uint sg, uint lane, threadgroup float* e, threadgroup float* sv, + threadgroup uint* si) { + const uint s = tg.y, t = sg * 32 + lane, id = prev[size_t(s) * p.prev_stride]; + for (uint i = t; i < p.rank; i += 256) e[i] = id < p.vocab ? float(w1[size_t(id) * p.rank + i]) : 0.0f; + threadgroup_barrier(mem_flags::mem_threadgroup); + const device bfloat* row = logits + size_t(s) * p.stride; + float mv = -INFINITY; + uint mi = 0xffffffffu; + bool nan = false; + const uint first = tg.x * MARKOV_SPAN, last = min(first + MARKOV_SPAN, p.vocab); + for (uint v = first + sg; v < last; v += 8) { + float d = 0.0f; + for (uint i = 8 * lane; i < p.rank; i += 256) { + if (BF16) { + float f[8]; + lfm2_unpack8(*(const device uint4*)((const device bfloat*)w2 + size_t(v) * p.rank + i), f); + for (int j = 0; j < 8; ++j) d = fma(f[j], e[i + j], d); + continue; + } + const size_t g = (size_t(v) * p.rank + i) / 64; + const float sc = w2s[g], bi = w2b[g]; + const uchar4 lo = *(const device uchar4*)(w2 + size_t(v) * p.rank + i); + const uchar4 hi = *(const device uchar4*)(w2 + size_t(v) * p.rank + i + 4); + for (int j = 0; j < 4; ++j) d = fma(fma(sc, float(lo[j]), bi), e[i + j], d); + for (int j = 0; j < 4; ++j) d = fma(fma(sc, float(hi[j]), bi), e[i + 4 + j], d); + } + d = simd_sum(d); + const float sc = float(static_cast(float(row[v]) + float(static_cast(d)))); + nan = nan || isnan(sc); + if (sc > mv || (mi == 0xffffffffu && sc == mv)) { + mv = sc; + mi = v; + } + } + if (lane == 0) { + sv[sg] = nan ? NAN : mv; + si[sg] = mi; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t != 0) return; + float bv = sv[0]; + uint bi = si[0]; + for (uint g = 1; g < 8; ++g) { + if (isnan(bv) || isnan(sv[g])) bv = NAN; + else if (sv[g] > bv || (sv[g] == bv && si[g] < bi)) { + bv = sv[g]; + bi = si[g]; + } + } + best[size_t(s) * grid.x + tg.x] = bv; + at[size_t(s) * grid.x + tg.x] = bi; +} + +[[kernel]] void lfm2_markov_scan(const device bfloat* logits [[buffer(0)]], const device bfloat* w1 [[buffer(1)]], + const device uint8_t* w2 [[buffer(2)]], const device uint* prev [[buffer(3)]], + constant lfm2_markov_args& p [[buffer(4)]], device float* best [[buffer(5)]], + device uint* at [[buffer(6)]], const device bfloat* w2s [[buffer(7)]], + const device bfloat* w2b [[buffer(8)]], uint2 tg [[threadgroup_position_in_grid]], + uint2 grid [[threadgroups_per_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + threadgroup float e[256]; + threadgroup float sv[8]; + threadgroup uint si[8]; + lfm2_markov_scan_t(logits, w1, w2, prev, p, best, at, w2s, w2b, tg, grid, sg, lane, e, sv, si); +} + +// The same scan over the released bf16 W2 [vocab, rank] (buffers 7 and 8 unused). +[[kernel]] void lfm2_markov_scan_bf16(const device bfloat* logits [[buffer(0)]], const device bfloat* w1 [[buffer(1)]], + const device uint8_t* w2 [[buffer(2)]], const device uint* prev [[buffer(3)]], + constant lfm2_markov_args& p [[buffer(4)]], device float* best [[buffer(5)]], + device uint* at [[buffer(6)]], uint2 tg [[threadgroup_position_in_grid]], + uint2 grid [[threadgroups_per_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + threadgroup float e[256]; + threadgroup float sv[8]; + threadgroup uint si[8]; + lfm2_markov_scan_t(logits, w1, w2, prev, p, best, at, w1, w1, tg, grid, sg, lane, e, sv, si); +} + +// DSpark's Markov step pick: the scan's best over a stream's threadgroups (ties to the lowest token), as a Pick. +[[kernel]] void lfm2_markov_pick(const device float* best [[buffer(0)]], const device uint* at [[buffer(1)]], + constant uint& groups [[buffer(2)]], device uint* picks [[buffer(3)]], + uint s [[threadgroup_position_in_grid]], uint t [[thread_index_in_threadgroup]]) { + threadgroup float tv[256]; + threadgroup uint ti[256]; + threadgroup uint tn[256]; + float bv = -INFINITY; + uint bi = 0xffffffffu, nan = 0; + for (uint g = t; g < groups; g += 256) { + const float v = best[size_t(s) * groups + g]; + const uint i = at[size_t(s) * groups + g]; + if (isnan(v)) nan = 1; + else if (v > bv || (v == bv && i < bi)) { + bv = v; + bi = i; + } + } + tv[t] = bv; + ti[t] = bi; + tn[t] = nan; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint w = 128; w > 0; w >>= 1) { + if (t < w) { + const float v = tv[t + w]; + const uint i = ti[t + w]; + if (v > tv[t] || (v == tv[t] && i < ti[t])) { + tv[t] = v; + ti[t] = i; + } + tn[t] |= tn[t + w]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (t == 0) { + picks[2 * s] = ti[0]; + picks[2 * s + 1] = tn[0]; + } +} + +// Held drafts from picks[i * stride] into ids[i]; a nonfinite or past-vocabulary pick is token 0 and sets bad. +[[kernel]] void lfm2_draft_ids(const device uint* picks [[buffer(0)]], constant uint4& p [[buffer(1)]], + device uint* ids [[buffer(2)]], device atomic_uint* bad [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + if (i >= p.y) return; + const uint token = picks[size_t(i) * p.x]; + const bool ok = token < p.z && picks[size_t(i) * p.x + 1] == 0u; + ids[i] = ok ? token : 0u; + if (!ok) atomic_store_explicit(bad, 1u, memory_order_relaxed); +} + +// DSpark's confidence logit of draft i of stream s: proj . [x row s * n + i, W1[previous token]] in fp32, no bias. +struct lfm2_confidence_args { uint n, pass, vocab, rank, hidden; }; +[[kernel]] void lfm2_dspark_confidence(const device bfloat* x [[buffer(0)]], const device bfloat* w1 [[buffer(1)]], + const device uint* ids [[buffer(2)]], const device uint* picks [[buffer(3)]], + const device bfloat* proj [[buffer(4)]], + constant lfm2_confidence_args& p [[buffer(5)]], device float* out [[buffer(6)]], + uint2 tg [[threadgroup_position_in_grid]], uint t [[thread_index_in_threadgroup]], + uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + threadgroup float part[8]; + const uint i = tg.x, s = tg.y; + // the previous token: the block's pending id for the first draft, else the Markov step's pick before it + const uint id = i == 0 ? ids[size_t(s) * p.n] : picks[2 * (size_t(i - 1) * p.pass + s)]; + const device bfloat* row = x + (size_t(s) * p.n + i) * p.hidden; + float d = 0.0f; + for (uint k = t; k < p.hidden; k += 256) d = fma(float(row[k]), float(proj[k]), d); + if (id < p.vocab) + for (uint k = t; k < p.rank; k += 256) d = fma(float(w1[size_t(id) * p.rank + k]), float(proj[p.hidden + k]), d); + d = simd_sum(d); + if (lane == 0) part[sg] = d; + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t != 0) return; + float sum = 0.0f; + for (uint g = 0; g < 8; ++g) sum += part[g]; + out[size_t(i) * p.pass + s] = sum; +} + +// DSpark's bf16 linears (the released matrices as stored): y [rows, n] = x [rows, k] W [n, k]^T, fp32 sums, bf16 out. +struct lfm2_bf16_args { uint rows, n, k; }; +constant constexpr uint BF16_ROWS = 16; + +// Few rows: a simdgroup a column of W over 16 rows, so W is read once a 16 rows. k a multiple of 8. +[[kernel]] void lfm2_bf16_rows(const device bfloat* x [[buffer(0)]], const device bfloat* w [[buffer(1)]], + constant lfm2_bf16_args& p [[buffer(2)]], device bfloat* y [[buffer(3)]], + uint2 tg [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + const uint col = tg.x * 8 + sg, r0 = tg.y * BF16_ROWS; + if (col >= p.n) return; + const uint rows = min(BF16_ROWS, p.rows - r0); + float acc[BF16_ROWS]; + for (uint r = 0; r < BF16_ROWS; ++r) acc[r] = 0.0f; + const device bfloat* wr = w + size_t(col) * p.k; + const device bfloat* xr = x + size_t(r0) * p.k; + for (uint i = 8 * lane; i < p.k; i += 256) { + float wv[8]; + lfm2_unpack8(*(const device uint4*)(wr + i), wv); + for (uint r = 0; r < BF16_ROWS; ++r) { + if (r >= rows) break; + float xv[8]; + lfm2_unpack8(*(const device uint4*)(xr + size_t(r) * p.k + i), xv); + for (int j = 0; j < 8; ++j) acc[r] = fma(wv[j], xv[j], acc[r]); + } + } + for (uint r = 0; r < BF16_ROWS; ++r) { + const float sum = simd_sum(acc[r]); + if (lane == 0 && r < rows) y[size_t(r0 + r) * p.n + col] = static_cast(sum); + } +} + +// Many rows: 64x64 tiles of y, four simdgroups of 32x32, k in steps of 32. k a multiple of 32. +[[kernel]] void lfm2_bf16_mma(const device bfloat* x [[buffer(0)]], const device bfloat* w [[buffer(1)]], + constant lfm2_bf16_args& p [[buffer(2)]], device bfloat* y [[buffer(3)]], + uint2 tg [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint t [[thread_index_in_threadgroup]]) { + threadgroup bfloat xs[64 * 32]; + threadgroup bfloat ws[64 * 32]; + threadgroup float out[4][32 * 32]; + const uint c0 = tg.x * 64, r0 = tg.y * 64; + simdgroup_matrix acc[4][4]; + for (uint i = 0; i < 4; ++i) + for (uint j = 0; j < 4; ++j) acc[i][j] = simdgroup_matrix(0.0f); + const uint sr = (sg / 2) * 32, sc = (sg % 2) * 32; + for (uint k0 = 0; k0 < p.k; k0 += 32) { + for (uint i = t; i < 64 * 4; i += 128) { // 64 rows of 32 bf16 as 4 chunks of 8, 16 bytes a load + const uint r = i / 4, q = (i % 4) * 8; + const uint4 xv = r0 + r < p.rows ? *(const device uint4*)(x + size_t(r0 + r) * p.k + k0 + q) : uint4(0); + const uint4 wv = c0 + r < p.n ? *(const device uint4*)(w + size_t(c0 + r) * p.k + k0 + q) : uint4(0); + *(threadgroup uint4*)(xs + r * 32 + q) = xv; + *(threadgroup uint4*)(ws + r * 32 + q) = wv; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint kk = 0; kk < 32; kk += 8) { + simdgroup_matrix a[4], b[4]; + for (uint i = 0; i < 4; ++i) simdgroup_load(a[i], xs + (sr + 8 * i) * 32 + kk, 32); + for (uint j = 0; j < 4; ++j) simdgroup_load(b[j], ws + (sc + 8 * j) * 32 + kk, 32, ulong2(0, 0), true); + for (uint i = 0; i < 4; ++i) + for (uint j = 0; j < 4; ++j) simdgroup_multiply_accumulate(acc[i][j], a[i], b[j], acc[i][j]); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + for (uint i = 0; i < 4; ++i) + for (uint j = 0; j < 4; ++j) simdgroup_store(acc[i][j], out[sg] + 8 * i * 32 + 8 * j, 32); + simdgroup_barrier(mem_flags::mem_threadgroup); + for (uint i = t % 32; i < 32 * 32; i += 32) { + const uint r = i / 32, c = i % 32; + const uint row = r0 + sr + r, col = c0 + sc + c; + if (row < p.rows && col < p.n) y[size_t(row) * p.n + col] = static_cast(out[sg][i]); + } +} + +// A drafter block's attention: lfm2_attn_mma where every row sees all keys, its 128 threads reserved for M1 and M2. +[[max_total_threads_per_threadgroup(128)]] [[kernel]] void lfm2_attn_block( + const device bfloat* queries [[buffer(0)]], const device bfloat* keys [[buffer(1)]], + const device bfloat* values [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]], device float* pm [[buffer(4)]], + device float* pl [[buffer(5)]], device float* po [[buffer(6)]], constant uint& rows [[buffer(7)]], + uint3 tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]], + uint sg [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]]) { + threadgroup uint ks[2][HD * (KT / 2)]; + threadgroup bfloat vs[2][KB * KP]; + lfm2_attn_rows<1, true>(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs); +} diff --git a/zig/kernels/metal/sources.zig b/zig/kernels/metal/sources.zig index 2506fb4905..9f4604f678 100644 --- a/zig/kernels/metal/sources.zig +++ b/zig/kernels/metal/sources.zig @@ -19,7 +19,7 @@ pub const gemma4_prompt_attention = @embedFile("gemma4/prompt_attention.metal"); pub const gemma4_vision = @embedFile("prefill/gemm_nax.metal") ++ @embedFile("gemma4/vision.metal"); /// LFM2-MoE: embedding, norms, the short conv, attention, the route and the experts' glue. -pub const lfm2 = @embedFile("lfm2/lfm2.metal") ++ @embedFile("lfm2/lfm2_segments.metal"); +pub const lfm2 = @embedFile("lfm2/lfm2.metal") ++ @embedFile("lfm2/lfm2_dspark.metal") ++ @embedFile("lfm2/lfm2_segments.metal"); /// LFM2's prompt attention on the tensor units (nax.h inlined at load, after lfm2.metal's text). pub const lfm2_attn_nax = @embedFile("lfm2/lfm2_attn_nax.metal"); diff --git a/zig/src/core/contract/lfm2_records.zig b/zig/src/core/contract/lfm2_records.zig index 355559b1ff..2e7587fa3d 100644 --- a/zig/src/core/contract/lfm2_records.zig +++ b/zig/src/core/contract/lfm2_records.zig @@ -1,12 +1,23 @@ //! LFM2-MoE's shared-round and greedy-pick launches (lfm2.metal, lfm2_segments.metal), as the encoders launch them. const std = @import("std"); const M = @import("limits.zig").MetalLaunch; +// The drafter kernels carry M1 and M2 limits too, recorded from the offline compile; no run covers those chips. pub const records = [_]M{ .{ .name = "metal lfm2 argmax lfm2_argmax", .threads = 1024, .tg_memory = 384, .m1 = 1024, .m2 = 1024 }, .{ .name = "metal lfm2 stream conv lfm2_conv_segs", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, .{ .name = "metal lfm2 stream attention qk lfm2_qk_segs", .threads = 32, .tg_memory = 0, .m1 = 832, .m2 = 832 }, .{ .name = "metal lfm2 stream attention lfm2_attn_segs", .threads = 128, .tg_memory = 18688, .reserved = 128 }, .{ .name = "metal lfm2 stream attention merge lfm2_attn_merge_segs", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 tap lfm2_tap", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 markov scan lfm2_markov_scan", .threads = 256, .tg_memory = 1088, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 markov scan bf16 lfm2_markov_scan_bf16", .threads = 256, .tg_memory = 1088, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 bf16 rows lfm2_bf16_rows", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 bf16 tile lfm2_bf16_mma", .threads = 128, .tg_memory = 24576, .m1 = 832, .m2 = 832 }, + .{ .name = "metal lfm2 markov pick lfm2_markov_pick", .threads = 256, .tg_memory = 3072, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 draft ids lfm2_draft_ids", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 block attention lfm2_attn_block", .threads = 128, .tg_memory = 18688, .reserved = 128 }, + .{ .name = "metal lfm2 block attention merge lfm2_attn_block_merge", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 dspark confidence lfm2_dspark_confidence", .threads = 256, .tg_memory = 32, .m1 = 1024, .m2 = 1024 }, }; test "the LFM2 launches fit every Apple family" { diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index 0f0905212f..38b1d9be0a 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -303,10 +303,12 @@ pub const Open = struct { /// --parallel named a number: an engine whose memory fits fewer streams refuses instead of serving fewer. lanes_fixed: bool = false, drafts: bool = true, - /// --drafter: a draft model's directory (Qwen3.8-27B's DFlash2; null: none), and --drafter-bits: 4, or 0 for bf16. + /// --drafter: a draft model's directory (null: none); --drafter-bits: 4, 0 for bf16, or 8 (LFM2 only). drafter: ?[]const u8 = null, drafter_name: ?[]const u8 = null, // the supplied --drafter name, before resolving its directory drafter_bits: u8 = 4, + /// Whether --drafter-bits was given: LFM2's drafter runs bf16 as released unless it is asked for 8-bit codes. + drafter_bits_given: bool = false, /// --record: each stream's committed rows and tapped features written under this folder, for drafter training. record: ?[]const u8 = null, speed_up: ?[]const u8 = null, diff --git a/zig/src/core/lanes/allocate.zig b/zig/src/core/lanes/allocate.zig index c3c592af60..d15b489c55 100644 --- a/zig/src/core/lanes/allocate.zig +++ b/zig/src/core/lanes/allocate.zig @@ -81,12 +81,17 @@ fn rate(costs: *const Table, timed: bool, total: u64, expected: f64, overhead_ms /// A chain's nodes' chances from per-depth acceptance (the j-th lands if every earlier one did). Caller frees. pub fn chainProbabilities(gpa: Allocator, rates: []const f64, count: usize) ![]f64 { const out = try gpa.alloc(f64, count); + chainProbabilitiesInto(rates, out); + return out; +} + +/// Fill a caller-owned chain buffer without allocating a second copy. +pub fn chainProbabilitiesInto(rates: []const f64, out: []f64) void { var reach: f64 = 1.0; for (out, 0..) |*o, j| { reach *= if (j < rates.len) rates[j] else rates[rates.len - 1]; o.* = reach; } - return out; } /// Costs for 1..rows: timed widths as measured, interpolated between them, the widest's ms a row past it. diff --git a/zig/src/core/lanes/config.zig b/zig/src/core/lanes/config.zig index a6721fcf57..deb76d4be2 100644 --- a/zig/src/core/lanes/config.zig +++ b/zig/src/core/lanes/config.zig @@ -36,7 +36,11 @@ pub const Model = struct { max_streams: u32 = 0, // a shared forward's streams at most (0: 32) shared_costs: []const Cost = &.{}, // shared forwards' ms by total rows (timed at load) draft_probabilities: bool = false, // the head gives each drafted node's chance of landing + granted_depth: bool = false, // with draft_probabilities: every stream drafts its last grant plus a slack + draft_slack: u32 = 0, // drafts past a stream's last grant, short or not (0: the Constants' own) draft_streams: bool = false, // the head drafts every stream of a shared round in one batch + copy_first: bool = false, // a copy ahead takes the round: the head holds no drafts for it (a drafter's own pass) + hold_costs: []const @import("hold_price.zig").HoldCost = &.{}, // a drafter's own hold pass, timed at load head_trees: bool = false, // the head drafts trees of lanes (DraftRequest.lanes) and holds them join_tail: bool = false, // a prompt in pieces leaves its last token for the next shared round's window (Backend.join) piece_runs: bool = false, // a round's pieces go to the backend in one call (Backend.pieces: one forward over them) @@ -66,6 +70,7 @@ pub const Config = struct { batch_rows: u32, batch_streams: u32, node_probabilities: bool, + granted_depth: bool = false, // Model.granted_depth fill_lanes: u32 = 0, // rows a window may grow to with suffix-match branches (0: off; lanes/fill.zig) fill_match: u32 = 8, // context tokens a suffix-match branch must match (shorter ones are coincidental phrases) fill_floor: f64 = 0.35, // the share of grafted rows a stream must keep to go on reading its chain back every round @@ -149,6 +154,7 @@ pub const Config = struct { .batch_rows = batch_rows, .batch_streams = if (streams) (if (m.max_streams != 0) m.max_streams else 32) else 1, .node_probabilities = m.draft_probabilities or m.draft_confidence != null, + .granted_depth = m.granted_depth and m.draft_probabilities, .draft_streams = m.draft_streams, .head_trees = m.head_trees, .join_tail = m.join_tail, @@ -160,7 +166,14 @@ pub const Config = struct { .shared_costs = try alloc.extendCosts(gpa, timed.items, batch_rows), .lane_costs = try alloc.extendCosts(gpa, m.lane_costs, 64), .mtp_step_ms = m.mtp_step_ms, - .k = .{ .depth_rate = if (m.depth_rate > 0) m.depth_rate else (Constants{}).depth_rate }, + .k = .{ + .depth_rate = if (m.depth_rate > 0) m.depth_rate else (Constants{}).depth_rate, + .draft_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).draft_slack, + .grant_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).grant_slack, + .short_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).short_slack, + .copy_first = m.copy_first and !m.context_copies, + .hold_costs = m.hold_costs, + }, .draft_step_ms = m.draft_step_ms, }; } diff --git a/zig/src/core/lanes/depth.zig b/zig/src/core/lanes/depth.zig index 8595a67aab..efb8dfa018 100644 --- a/zig/src/core/lanes/depth.zig +++ b/zig/src/core/lanes/depth.zig @@ -12,8 +12,15 @@ pub const Constants = struct { plain_margin: f64 = 1.05, // a drafted depth must beat plain rounds' tokens a ms by this much plain_wait_most: u64 = 128, // plain rounds between probes at most draft_slack: i64 = 2, // drafts a stream offers past what its last shared round granted it + grant_slack: i64 = 1, // granted depths: drafts past a grant that verified every draft the stream held + short_slack: i64 = 0, // granted depths: drafts past a grant that left some of them unverified + copy_first: bool = false, // a copy ahead takes the next round whole: the head drafts none for it (else one) + hold_costs: []const HoldCost = &.{}, // a drafter's own hold pass by blocks and depth: granted holds must pay for it }; +pub const HoldCost = @import("hold_price.zig").HoldCost; +const holdMs = @import("hold_price.zig").holdMs; + /// A stream's acceptance by depth and probe counters (Python `_depth_state[stream]`). pub const State = struct { p: []f64, @@ -35,6 +42,8 @@ pub const Who = struct { finished: bool, forced: bool, granted: ?u32 = null, + grant_short: bool = false, // the last grant left some held drafts unverified (the width did not bind) + gain: ?f64 = null, // the drafts its last grant expected to keep (null: none priced) stream: ?*anyopaque = null, sampled: bool = false, // keyed draws: host work a row that a greedy round does not have }; @@ -59,6 +68,7 @@ pub const Rule = struct { draft_step_ms: f64 = 0, // a shared round's draft past a stream's first (budgets price it) plain_guard: bool, node_probabilities: bool, + granted_depth: bool = false, // node_probabilities: a stream drafts its last grant plus draft_slack, lone ones too confidence_floor: ?f64 = null, batch_rows: u32, batched_heads: bool = false, // one batch drafts every stream of a shared round (its steps: the deepest stream's) @@ -68,7 +78,10 @@ pub const Rule = struct { shared_rounds: u64 = 0, greedy_overhead: std.ArrayList(Overhead) = .empty, // overhead from rounds without a sampled stream + hold_order: []usize = &.{}, // reused hold ordering, at most batch_rows entries + pub fn deinit(r: *Rule) void { + r.gpa.free(r.hold_order); r.round_ms.deinit(r.gpa); r.overhead.deinit(r.gpa); r.greedy_ms.deinit(r.gpa); @@ -192,17 +205,105 @@ pub const Rule = struct { return true; } - /// Python `_head_depth`: the next round's head drafts for a stream (0 when forced, 1 with a copy ahead). + /// Drafts past a grant: none when it stopped short of the held drafts (the width did not bind), else grant_slack. + fn slack(r: *const Rule, who: Who) i64 { + return if (who.grant_short) r.k.short_slack else r.k.grant_slack; + } + + /// The last grant plus its slack (the deepest chain before any); after a grant of none, a probe of draft_slack. + fn grantDepth(r: *Rule, who: Who) !i64 { + const most = @min(@as(i64, r.most_drafts), @max(1, who.draft_room - 1)); + const granted = who.granted orelse return most; + _ = try r.rates(who); + const st = &who.state.*.?; + if (granted == 0) return if (!r.plain_guard or r.probePlain(st)) @min(most, r.k.draft_slack) else 0; + st.plain = 0; + st.wait = r.k.depth_probe_every; + return @min(most, @as(i64, granted) + r.slack(who)); + } + + /// Python `_head_depth`: the next round's head drafts for a stream (0 when forced, 1 or none with a copy ahead). pub fn headDepth(r: *Rule, who: *const Who, budget: ?i64, probe: Probe) !i64 { - const d: i64 = if (who.finished) 0 else if (budget) |b| b else if (r.node_probabilities) + const d: i64 = if (who.finished) 0 else if (budget) |b| b else if (r.granted_depth) + try r.grantDepth(who.*) + else if (r.node_probabilities) @min(@as(i64, r.most_drafts), @max(1, who.draft_room - 1)) else try r.depth(who.*); if (who.forced) return 0; - if (d != 0 and probe.copy(probe.ptr, who)) return 1; + if (d != 0 and probe.copy(probe.ptr, who)) return @intFromBool(!r.k.copy_first); return d; } + /// A shared hold pads every block to its deepest: depths cut so the drafting streams' blocks fit `rows` (one pass). + fn capPass(out: []i64, rows: u32) void { + var n: i64 = 0; + for (out) |d| n += @intFromBool(d > 0); + if (n == 0) return; + const most = @max(1, @divFloor(@as(i64, rows), n)); + for (out) |*d| d.* = @min(d.*, most); + } + + /// Granted holds kept, most expected drafts a row first, while they raise the next round's expected tokens a ms. + fn payHolds(r: *Rule, whos: []const Who, sampled: bool, out: []i64) !void { + if (whos.len > r.batch_rows) return error.BadHoldStreamCount; + if (r.hold_order.len == 0) r.hold_order = try r.gpa.alloc(usize, r.batch_rows); + const order = r.hold_order[0..whos.len]; + var m: usize = 0; + var probes: usize = 0; // a probe after a grant of none keeps its hold: it measures the stream again + var deepest: i64 = 0; + for (whos, out, 0..) |w, d, i| { + if (d <= 0) continue; + if (w.granted == @as(?u32, 0)) { + probes += 1; + deepest = @max(deepest, d); + } else { + order[m] = i; + m += 1; + } + } + if (m == 0) return; + const Ctx = struct { + r: *Rule, + whos: []const Who, + fn less(c: @This(), x: usize, y: usize) bool { + const gx = c.r.gainOf(c.whos[x]) catch 0; + const gy = c.r.gainOf(c.whos[y]) catch 0; + return gx * rowsOf(c.whos[y]) > gy * rowsOf(c.whos[x]) or (gx * rowsOf(c.whos[y]) == gy * rowsOf(c.whos[x]) and x < y); + } + }; + std.mem.sort(usize, order[0..m], Ctx{ .r = r, .whos = whos }, Ctx.less); + const fixed = r.fixedMs(whos.len, sampled); + var rows: f64 = @floatFromInt(whos.len); + var tokens = rows; + const plain = r.shared_costs.get(@intCast(whos.len)) orelse return; + const probe_ms = if (probes > 0) holdMs(r.k.hold_costs, probes, deepest) else 0; + var best = tokens / (plain + fixed + probe_ms); + var keep: usize = 0; + for (order[0..m], 1..) |i, k| { + rows += rowsOf(whos[i]); + tokens += try r.gainOf(whos[i]); + deepest = @max(deepest, out[i]); + const verify = r.shared_costs.get(@intFromFloat(@ceil(rows))) orelse break; + const now = tokens / (verify + fixed + holdMs(r.k.hold_costs, k + probes, deepest)); + if (now > best) { + best = now; + keep = k; + } + } + for (order[keep..m]) |i| out[i] = 0; + } + + /// The drafts a granted stream's next round is expected to keep: its last grant's, or its first draft's rate. + fn gainOf(r: *Rule, w: Who) !f64 { + if (w.gain) |g| if (w.granted != null) return g; + return (try r.rates(w))[0]; + } + + fn rowsOf(w: Who) f64 { + return @floatFromInt(@max(1, w.granted orelse 1)); + } + /// Head drafts for the streams of a shared round, sized by allocation over their chain chances. pub fn budgets(r: *Rule, whos: []Who, probe: Probe, sampled: bool, out: []i64) !void { if (r.confidence_floor != null) { @@ -213,8 +314,10 @@ pub const Rule = struct { for (whos, out) |*w, *o| { const inner = try r.headDepth(w, null, probe); const granted: i64 = if (w.granted) |g| g else r.most_drafts; - o.* = try r.headDepth(w, @min(inner, granted + r.k.draft_slack), probe); + o.* = if (r.granted_depth) inner else try r.headDepth(w, @min(inner, granted + r.k.draft_slack), probe); } + if (r.granted_depth) capPass(out, r.batch_rows); + if (r.granted_depth and r.k.hold_costs.len > 0) try r.payHolds(whos, sampled, out); return; } const most = try r.gpa.alloc(i64, whos.len); @@ -381,3 +484,88 @@ test "a shared round's drafts come out the same whichever depths its overhead wa } try std.testing.expectEqualSlices(i64, &got[0], &got[1]); } + +test "granted depths in a shared round fit one hold pass: every block is as deep as the deepest" { + const empty: Table = .{}; + var rule: Rule = .{ .gpa = std.testing.allocator, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = false, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + var states: [8]?State = @splat(null); + defer for (&states) |*st| if (st.*) |*x| x.deinit(std.testing.allocator); + var whos: [8]Who = undefined; + for (&whos, &states, 0..) |*w, *st, i| w.* = .{ .state = st, .draft_room = 100, .finished = false, .forced = false, .granted = @intCast(i) }; + const No = struct { + fn copy(_: *anyopaque, _: *const Who) bool { + return false; + } + }; + var unused: u8 = 0; + var out: [8]i64 = undefined; + try rule.budgets(&whos, .{ .ptr = &unused, .copy = No.copy }, false, &out); + // a grant of none probes two; grants 1..7 that took every held draft, plus one; cut to 32 rows over eight blocks + try std.testing.expectEqualSlices(i64, &.{ 2, 2, 3, 4, 4, 4, 4, 4 }, &out); + try rule.budgets(whos[0..2], .{ .ptr = &unused, .copy = No.copy }, false, out[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 2, 2 }, out[0..2]); // two blocks fit whole + for (whos[0..2]) |*w| w.grant_short = true; + try rule.budgets(whos[0..2], .{ .ptr = &unused, .copy = No.copy }, false, out[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 2, 1 }, out[0..2]); // a grant short of the held drafts: no slack +} + +test "a copy ahead: one head draft beside it, or none when the head's drafts cost a pass of their own" { + const empty: Table = .{}; + const Yes = struct { + fn copy(_: *anyopaque, _: *const Who) bool { + return true; + } + }; + var unused: u8 = 0; + const probe: Probe = .{ .ptr = &unused, .copy = Yes.copy }; + var st: ?State = null; + defer if (st) |*x| x.deinit(std.testing.allocator); + const who: Who = .{ .state = &st, .draft_room = 100, .finished = false, .forced = false, .granted = 3 }; + for ([_]bool{ false, true }) |first| { + var rule: Rule = .{ .gpa = std.testing.allocator, .k = .{ .copy_first = first }, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = false, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + try std.testing.expectEqual(@as(i64, @intFromBool(!first)), try rule.headDepth(&who, null, probe)); + var out: [1]i64 = undefined; + var whos = [_]Who{who}; + try rule.budgets(&whos, probe, false, &out); + try std.testing.expectEqual(@as(i64, @intFromBool(!first)), out[0]); + } +} + +test "a granted hold stays only while its stream's expected drafts pay for the verify rows and the hold pass" { + const gpa = std.testing.allocator; + var shared: Table = .{}; + defer shared.deinit(gpa); + for (1..33) |w| try shared.put(gpa, @intCast(w), 10.0 + @as(f64, @floatFromInt(w))); + const empty: Table = .{}; + const cheap = [_]HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1.0 }, .{ .streams = 8, .depth = 1, .ms = 4.5 } }; + var states: [4]?State = @splat(null); + defer for (&states) |*st| if (st.*) |*x| x.deinit(gpa); + const granted = [_]?u32{ 2, 2, 0, 1 }; + const gains = [_]?f64{ 1.8, 0.2, null, 0.9 }; + var whos: [4]Who = undefined; + for (&whos, &states, granted, gains) |*w, *st, g, x| w.* = .{ .state = st, .draft_room = 100, .finished = false, .forced = false, .granted = g, .gain = x }; + var rule: Rule = .{ .gpa = gpa, .k = .{ .hold_costs = &cheap }, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &shared, .mtp_step_ms = 0.5, .plain_guard = true, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + // 8 ms overhead: plain 4 / 22; stream 0 then 3 raise the rate (5.8 / 25, 6.7 / 26.5), stream 1 would lower it + var out = [_]i64{ 4, 4, 0, 3 }; + try rule.payHolds(&whos, false, &out); + try std.testing.expectEqualSlices(i64, &.{ 4, 0, 0, 3 }, &out); + var failing = std.testing.FailingAllocator.init(gpa, .{ .fail_index = 0 }); + rule.gpa = failing.allocator(); + out = .{ 4, 4, 0, 3 }; + try rule.payHolds(&whos, false, &out); + try std.testing.expect(!failing.has_induced_failure); + try std.testing.expectEqualSlices(i64, &.{ 4, 0, 0, 3 }, &out); + rule.gpa = gpa; + const dear = [_]HoldCost{.{ .streams = 1, .depth = 1, .ms = 20.0 }}; + rule.k.hold_costs = &dear; + out = .{ 4, 4, 0, 3 }; + try rule.payHolds(&whos, false, &out); + try std.testing.expectEqualSlices(i64, &.{ 0, 0, 0, 0 }, &out); // no gain pays a 20 ms pass + whos[2].granted = 0; + out = .{ 4, 4, 2, 3 }; + try rule.payHolds(&whos, false, &out); + try std.testing.expectEqualSlices(i64, &.{ 0, 0, 2, 0 }, &out); // a probe after a grant of none keeps its hold +} diff --git a/zig/src/core/lanes/drafted.zig b/zig/src/core/lanes/drafted.zig index b7b9ca5bc6..6c9f7edc4b 100644 --- a/zig/src/core/lanes/drafted.zig +++ b/zig/src/core/lanes/drafted.zig @@ -105,6 +105,15 @@ pub const Drafted = struct { m.draft_streams = d.batched; m.draft_probabilities = false; m.head_trees = d.trees; + m.copy_first = d.copy_first; + m.hold_costs = d.hold_costs; + if (d.chances and x.drafter.vtable.chances != null) { + // a chain's drafts priced by the drafter's own chances; the next hold as deep as the grant and its slack + m.draft_probabilities = d.confidence_floor == null; + m.draft_confidence = d.confidence_floor; + m.granted_depth = d.confidence_floor == null; + m.draft_slack = d.slack; + } if (d.tree_nodes > 0) { // the drafter grows its own trees, best first, and prices their nodes m.drafts = @min(d.tree_nodes, if (target.exact_width > 1) target.exact_width - 1 else 0); diff --git a/zig/src/core/lanes/drafted_chances_test.zig b/zig/src/core/lanes/drafted_chances_test.zig new file mode 100644 index 0000000000..233ec38fe5 --- /dev/null +++ b/zig/src/core/lanes/drafted_chances_test.zig @@ -0,0 +1,287 @@ +//! A drafter that prices its chains (Facts.chances): verify rows go by its chances and replies stay one-token decode. +const std = @import("std"); +const Config = @import("config.zig").Config; +const Model = @import("config.zig").Model; +const Cost = @import("config.zig").Cost; +const Engine = @import("engine.zig").Engine; +const be = @import("backend.zig"); +const sm = @import("stream.zig"); +const fake = @import("fake.zig"); +const dr = @import("drafter.zig"); +const depth = @import("depth.zig"); +const Table = @import("table.zig").Table; +const Drafted = @import("drafted.zig").Drafted; +const dt = @import("drafted_test.zig"); + +const gpa = std.testing.allocator; +const most_streams = 8; + +/// drafted_test's drafter with a chance for each held draft: `sure` where its guess is right, `unsure` where wrong. +const Priced = struct { + head: dt.FakeDrafter, + sure: f64 = 0.7, + unsure: f64 = 0.7, + vtable: dr.Drafter.VTable = undefined, + inner_hold: *const fn (*anyopaque, []const dr.Hold) anyerror!void = undefined, + at: std.AutoHashMapUnmanaged(*sm.Stream, u64) = .empty, // each stream's last hold's position + asked: u32 = 0, + held: u64 = 0, // drafts its holds asked for + + fn of(ptr: *anyopaque) *Priced { + return @fieldParentPtr("head", @as(*dt.FakeDrafter, @ptrCast(@alignCast(ptr)))); + } + + fn hold(ptr: *anyopaque, items: []const dr.Hold) anyerror!void { + const p = of(ptr); + for (items) |it| { + try p.at.put(gpa, it.stream, it.position); + p.held += it.depth; + } + return p.inner_hold(ptr, items); + } + + /// The running product along the chain, as the DSpark confidence head's survivals are. + fn chances(ptr: *anyopaque, s: *sm.Stream, out: []f64) anyerror!bool { + const p = of(ptr); + const lane = p.head.lanes.getPtr(s) orelse return false; + if (out.len > lane.held.items.len) return error.TooFewDrafts; + const position = p.at.get(s) orelse return error.NoHold; + p.asked += 1; + var reach: f64 = 1; + for (out, 0..) |*o, j| { + const at = position + j; + reach *= if ((at *% 2654435761 + j) % 5 == 0) p.unsure else p.sure; // drafted_test's wrong guesses + o.* = reach; + } + return true; + } + + fn drafter(p: *Priced) dr.Drafter { + const base_ = p.head.drafter(); + p.vtable = base_.vtable.*; + p.inner_hold = base_.vtable.hold; + p.vtable.hold = hold; + p.vtable.chances = chances; + return .{ .ptr = &p.head, .vtable = &p.vtable }; + } + + fn deinit(p: *Priced) void { + p.at.deinit(gpa); + p.head.deinit(); + } +}; + +/// Every shared verify's window rows: whether one round gave its streams unequal rows (a ragged verify). +const Seen = struct { + var inner: *const fn (*anyopaque, []const be.Window, []be.Verified) anyerror!void = undefined; + var ragged: u32 = 0; + var rows: u64 = 0; + + fn verify(ptr: *anyopaque, windows: []const be.Window, out: []be.Verified) anyerror!void { + for (windows) |w| { + rows += w.rows(); + if (w.rows() != windows[0].rows()) { + ragged += 1; + break; + } + } + return inner(ptr, windows, out); + } +}; + +var costs: [16]Cost = undefined; + +fn base() Model { + for (&costs, 1..) |*c, w| c.* = .{ .width = @intCast(w), .ms = 5.0 + 0.8 * @as(f64, @floatFromInt(w)) }; + return .{ .exact_width = 16, .gpu_tokens = true, .window_costs = &costs, .hidden_rows = true, .batch_rows = 32, .max_streams = most_streams }; +} + +const Run = struct { tokens: [most_streams][64]u32 = undefined, lens: [most_streams]usize = @splat(0), drafted: u64 = 0, holds: u32 = 0, asked: u32 = 0, ragged: u32 = 0, rows: u64 = 0, held: u64 = 0 }; + +/// The drafter's Facts.slack for the next runs (0: the depth rule's own; 9: every hold the whole block). +var slack: u32 = 0; + +/// The drafter's chances: equal everywhere, or knowing its wrong guesses (a confidence head that ranks well). +const Head = enum { flat, knows }; + +/// Each prompt decoded together (one stream: alone); `floor` null: the chances price rows, else a fixed window. +fn run(prompts: []const []const u32, drafts: bool, floor: ?f64, head: Head) !Run { + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var p: Priced = .{ .head = .{ .facts_ = .{ .depth = 9, .step_ms = 0.5, .batched = true, .chances = true, .confidence_floor = floor, .slack = slack } } }; + if (head == .knows) { + p.sure = 0.95; + p.unsure = 0.05; + } + defer p.deinit(); + var strict: dt.Strict = .{ .inner = target.backend() }; + defer strict.deinit(); + var seen = strict.backend(); + var table = seen.vtable.*; + Seen.inner = table.verify; + Seen.ragged = 0; + Seen.rows = 0; + table.verify = Seen.verify; + seen.vtable = &table; + var wrapped = try Drafted.init(gpa, seen, p.drafter()); + defer wrapped.deinit(); + var cfg = try Config.init(gpa, if (drafts) wrapped.facts(base()) else base(), 16, 15); + defer cfg.deinit(gpa); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, if (drafts) wrapped.backend() else target.backend(), clock.clock()); + defer engine.deinit(); + var streams: [most_streams]sm.Stream = undefined; + const n = prompts.len; + for (prompts, streams[0..n]) |prompt, *s| s.* = try sm.Stream.init(gpa, .{ .id = "s", .prompt = prompt, .max_new = 48, .eos = &.{96}, .drafts = drafts }); + defer for (streams[0..n]) |*s| s.deinit(gpa); + for (streams[0..n]) |*s| try engine.addStream(s); + while (engine.activeCount() > 0) try engine.step(); + var out: Run = .{ .holds = p.head.holds, .asked = p.asked, .ragged = Seen.ragged, .rows = Seen.rows, .held = p.held }; + for (streams[0..n], 0..) |*s, i| { + out.lens[i] = s.emitted().len; + @memcpy(out.tokens[i][0..out.lens[i]], s.emitted()); + out.drafted += s.drafted; + } + return out; +} + +fn expectSame(a: Run, b: Run, n: usize) !void { + for (0..n) |i| try std.testing.expectEqualSlices(u32, a.tokens[i][0..a.lens[i]], b.tokens[i][0..b.lens[i]]); +} + +/// `n` streams of unequal prompts, each a prefix of drafted_test's long prompt. +fn unequal(buf: *[most_streams][]const u32, n: usize) []const []const u32 { + for (buf[0..n], 0..) |*p, i| p.* = dt.long[0 .. 11 + i]; + return buf[0..n]; +} + +test "a drafter's chances price its chain's rows, and drafted replies equal one-token decode alone and shared" { + const sets = [_][]const []const u32{ &.{&dt.p1}, &.{ &dt.p1, &dt.long, &dt.p1 } }; + for (sets) |ps| { + const plain = try run(ps, false, null, .flat); + const priced = try run(ps, true, null, .flat); + try expectSame(plain, priced, ps.len); + try std.testing.expect(priced.asked > 0 and priced.drafted > 0); // the window allocator read the chances + const fixed = try run(ps, true, 0, .flat); + try expectSame(plain, fixed, ps.len); + try std.testing.expect(fixed.drafted > priced.drafted); // a fixed window verifies every draft the rows fit + } +} + +test "at 1, 4 and 8 streams the scheduled and the fixed verify equal one-token decode; shared rounds are ragged" { + var buf: [most_streams][]const u32 = undefined; + for ([_]usize{ 1, 4, 8 }) |n| { + const ps = unequal(&buf, n); + const plain = try run(ps, false, null, .flat); + const scheduled = try run(ps, true, null, .knows); + try expectSame(plain, scheduled, n); + const fixed = try run(ps, true, 0, .knows); + try expectSame(plain, fixed, n); + try std.testing.expect(scheduled.asked > 0 and scheduled.drafted > 0); + // a confident head's survivals cut each stream's chain where its guess goes wrong: fewer rows verified + try std.testing.expect(scheduled.rows < fixed.rows); + if (n > 1) try std.testing.expect(scheduled.ragged > 0); // streams' verify lengths differ within a round + } +} + +test "the drafter's chances reach the lane facts: rows by chances and holds by grants, or a fixed window" { + var p: Priced = .{ .head = .{ .facts_ = .{ .depth = 9, .step_ms = 0.5, .chances = true } } }; + defer p.deinit(); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var wrapped: Drafted = .{ .gpa = gpa, .target = target.backend(), .drafter = p.drafter() }; + defer wrapped.deinit(); + const m = wrapped.facts(base()); + try std.testing.expect(m.draft_probabilities and m.granted_depth and m.draft_confidence == null); + var cfg = try Config.init(gpa, m, 16, 15); + defer cfg.deinit(gpa); + try std.testing.expectEqual(@as(i64, 2), cfg.k.draft_slack); // the depth rule's own + try std.testing.expect(cfg.k.grant_slack == 1 and cfg.k.short_slack == 0); // a granted width's own + p.head.facts_.slack = 1; + var one = try Config.init(gpa, wrapped.facts(base()), 16, 15); + defer one.deinit(gpa); + try std.testing.expectEqual(@as(i64, 1), one.k.draft_slack); // a drafter's own slack past its grant + try std.testing.expect(one.k.grant_slack == 1 and one.k.short_slack == 1); // short or not + p.head.facts_.confidence_floor = 0; + const f = wrapped.facts(base()); + try std.testing.expect(!f.draft_probabilities and !f.granted_depth and f.draft_confidence.? == 0); + p.head.facts_.chances = false; + const none = wrapped.facts(base()); + try std.testing.expect(!none.draft_probabilities and none.draft_confidence == null); +} + +test "a granted depth: the deepest chain first, then the grant plus its slack, and after a grant of none a probe" { + const empty: Table = .{}; + var rule: depth.Rule = .{ .gpa = gpa, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = true, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + var st: ?depth.State = null; + defer if (st) |*s| s.deinit(gpa); + const No = struct { + fn copy(_: *anyopaque, _: *const depth.Who) bool { + return false; + } + }; + var unused: u8 = 0; + const probe: depth.Probe = .{ .ptr = &unused, .copy = No.copy }; + var who: depth.Who = .{ .state = &st, .draft_room = 100, .finished = false, .forced = false }; + try std.testing.expectEqual(@as(i64, 9), try rule.headDepth(&who, null, probe)); + who.granted = 3; + try std.testing.expectEqual(@as(i64, 4), try rule.headDepth(&who, null, probe)); // every held draft granted: one more + who.grant_short = true; + try std.testing.expectEqual(@as(i64, 3), try rule.headDepth(&who, null, probe)); // drafts left unverified: the grant + who.grant_short = false; + who.granted = 8; + try std.testing.expectEqual(@as(i64, 9), try rule.headDepth(&who, null, probe)); + rule.k.grant_slack = 2; // a drafter's slack sets both + rule.k.short_slack = 2; + who.grant_short = true; + who.granted = 3; + try std.testing.expectEqual(@as(i64, 5), try rule.headDepth(&who, null, probe)); + who.granted = 0; + var drafted: usize = 0; + for (0..8) |_| drafted += @intFromBool(try rule.headDepth(&who, null, probe) > 0); + try std.testing.expectEqual(@as(usize, 1), drafted); // one probe of draft_slack drafts in the first eight rounds +} + +test "holds as wide as the grants: replies equal one-token decode at 1, 4 and 8 streams, fewer drafts than whole blocks" { + var buf: [most_streams][]const u32 = undefined; + defer slack = 0; + for ([_]usize{ 1, 4, 8 }) |n| { + const ps = unequal(&buf, n); + const plain = try run(ps, false, null, .flat); + for ([_]Head{ .flat, .knows }) |head| { + slack = 0; + const narrow = try run(ps, true, null, head); + slack = 2; + const two = try run(ps, true, null, head); + slack = 9; + const whole = try run(ps, true, null, head); + try expectSame(plain, narrow, n); + try expectSame(plain, two, n); + try expectSame(plain, whole, n); + try std.testing.expect(narrow.drafted > 0); + try std.testing.expect(narrow.held < two.held and two.held < whole.held); + } + } +} + +test "hold budgeting queries a stateful copy proposer once for a granted depth" { + const empty: Table = .{}; + var rule: depth.Rule = .{ .gpa = gpa, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 1, .plain_guard = true, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + var state: ?depth.State = null; + defer if (state) |*s| s.deinit(gpa); + var whos = [_]depth.Who{.{ .state = &state, .draft_room = 64, .finished = false, .forced = false, .granted = 1 }}; + const Changing = struct { + fn copy(ptr: *anyopaque, _: *const depth.Who) bool { + const calls: *usize = @ptrCast(@alignCast(ptr)); + calls.* += 1; + return calls.* > 1; + } + }; + var calls: usize = 0; + var out: [1]i64 = undefined; + try rule.budgets(&whos, .{ .ptr = &calls, .copy = Changing.copy }, false, &out); + try std.testing.expectEqual(@as(usize, 1), calls); + try std.testing.expectEqual(@as(i64, 2), out[0]); +} diff --git a/zig/src/core/lanes/drafted_copy_test.zig b/zig/src/core/lanes/drafted_copy_test.zig new file mode 100644 index 0000000000..329731f69c --- /dev/null +++ b/zig/src/core/lanes/drafted_copy_test.zig @@ -0,0 +1,135 @@ +//! A drafter whose hold costs a pass of its own: no hold for a stream whose next round is a copy. +const std = @import("std"); +const Config = @import("config.zig").Config; +const Engine = @import("engine.zig").Engine; +const be = @import("backend.zig"); +const sm = @import("stream.zig"); +const fake = @import("fake.zig"); +const dr = @import("drafter.zig"); +const Proposer = @import("proposer.zig").Proposer; +const Drafted = @import("drafted.zig").Drafted; +const dt = @import("drafted_test.zig"); + +const gpa = std.testing.allocator; + +/// Copies the fake target's own continuation (it lands whole) whenever the context's length is a multiple of three. +const Copier = struct { + out: [3]u32 = undefined, + match: i64 = 0, + + fn propose(ptr: *anyopaque, context: []const u32, most: i64) anyerror![]const u32 { + const c: *Copier = @ptrCast(@alignCast(ptr)); + c.match = 0; + if (context.len % 3 != 0 or most < 2) return &.{}; + var history: [128]u32 = undefined; + @memcpy(history[0..context.len], context); + const n: usize = @intCast(@min(3, most)); + for (c.out[0..n], 0..) |*t, k| { + t.* = fake.next(history[0 .. context.len + k], null, 0); + history[context.len + k] = t.*; + } + c.match = 100; + return c.out[0..n]; + } + + fn lastMatch(ptr: *anyopaque) i64 { + return @as(*Copier, @ptrCast(@alignCast(ptr))).match; + } + + const vtable: Proposer.VTable = .{ .propose = propose, .last_match = lastMatch }; +}; + +/// Each stream's last hold position, and the copy windows verified at a position a hold was asked for. +const Watch = struct { + var inner_hold: *const fn (*anyopaque, []const dr.Hold) anyerror!void = undefined; + var inner_verify: *const fn (*anyopaque, []const be.Window, []be.Verified) anyerror!void = undefined; + var at: [4]?u64 = @splat(null); + var streams: [4]?*sm.Stream = @splat(null); + var wasted: u32 = 0; + var copies: u32 = 0; + var blocks: u32 = 0; // the streams' blocks held, over every hold + + fn slot(s: *sm.Stream) usize { + for (&streams, 0..) |*x, i| if (x.* == null or x.* == s) { + x.* = s; + return i; + }; + unreachable; + } + + fn hold(ptr: *anyopaque, items: []const dr.Hold) anyerror!void { + blocks += @intCast(items.len); + // a joining stream's first hold comes before its first token is read: no copy can be foreseen for it + for (items) |it| at[slot(it.stream)] = if (it.stream.emitted().len > 0) it.position else null; + return inner_hold(ptr, items); + } + + fn verify(ptr: *anyopaque, windows: []const be.Window, out: []be.Verified) anyerror!void { + for (windows) |w| if (w.held == 0 and w.tokens.len > 0) { + copies += 1; + if (at[slot(w.stream)] == w.positions[0]) wasted += 1; + }; + return inner_verify(ptr, windows, out); + } +}; + +const Run = struct { tokens: [3][64]u32 = undefined, lens: [3]usize = @splat(0), holds: u32 = 0, wasted: u32 = 0, copies: u32 = 0 }; + +fn run(n: usize, drafts: bool, copy_first: bool) !Run { + Watch.at = @splat(null); + Watch.streams = @splat(null); + Watch.wasted = 0; + Watch.copies = 0; + Watch.blocks = 0; + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var head: dt.FakeDrafter = .{ .facts_ = .{ .depth = 4, .step_ms = 0.5, .batched = true, .copy_first = copy_first } }; + defer head.deinit(); + var hooked = head.drafter(); + var table = hooked.vtable.*; + Watch.inner_hold = table.hold; + table.hold = Watch.hold; + hooked.vtable = &table; + var wrapped = try Drafted.init(gpa, target.backend(), hooked); + defer wrapped.deinit(); + // the round loop's windows, before the wrapper turns held drafts into host tokens + var seen = if (drafts) wrapped.backend() else target.backend(); + var verified = seen.vtable.*; + Watch.inner_verify = verified.verify; + verified.verify = Watch.verify; + seen.vtable = &verified; + const m: @import("config.zig").Model = .{ .exact_width = 16, .gpu_tokens = true, .hidden_rows = true, .batch_rows = 32, .max_streams = 4 }; + var cfg = try Config.init(gpa, if (drafts) wrapped.facts(m) else m, 16, 15); + defer cfg.deinit(gpa); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, seen, clock.clock()); + defer engine.deinit(); + var copiers: [3]Copier = @splat(.{}); + var streams: [3]sm.Stream = undefined; + for (streams[0..n], copiers[0..n], 0..) |*s, *c, i| s.* = try sm.Stream.init(gpa, .{ .id = "s", .prompt = dt.long[0 .. 12 + i], .max_new = 40, .eos = &.{96}, .drafts = drafts, .proposer = .{ .ptr = c, .vtable = &Copier.vtable } }); + defer for (streams[0..n]) |*s| s.deinit(gpa); + for (streams[0..n]) |*s| try engine.addStream(s); + while (engine.activeCount() > 0) try engine.step(); + var out: Run = .{ .holds = Watch.blocks, .wasted = Watch.wasted, .copies = Watch.copies }; + for (streams[0..n], 0..) |*s, i| { + out.lens[i] = s.emitted().len; + @memcpy(out.tokens[i][0..out.lens[i]], s.emitted()); + } + return out; +} + +test "a copy ahead takes the round: with copy_first no hold is asked for it, and replies equal one-token decode" { + for ([_]usize{ 1, 3 }) |n| { + const plain = try run(n, false, false); + const before = try run(n, true, false); + const after = try run(n, true, true); + for (0..n) |i| { + try std.testing.expectEqualSlices(u32, plain.tokens[i][0..plain.lens[i]], before.tokens[i][0..before.lens[i]]); + try std.testing.expectEqualSlices(u32, plain.tokens[i][0..plain.lens[i]], after.tokens[i][0..after.lens[i]]); + } + try std.testing.expect(before.copies > 0 and after.copies > 0); + try std.testing.expect(before.wasted > 0); // a hold of one draft beside each copy, dropped by the copy round + try std.testing.expectEqual(@as(u32, 0), after.wasted); + try std.testing.expect(after.holds < before.holds); + } +} diff --git a/zig/src/core/lanes/drafter.zig b/zig/src/core/lanes/drafter.zig index 2d82d40d04..8ecd22db47 100644 --- a/zig/src/core/lanes/drafter.zig +++ b/zig/src/core/lanes/drafter.zig @@ -18,6 +18,11 @@ pub const Facts = struct { window_costs: []const Cost = &.{}, // the target's window ms by rows (timed at load), for the depth rule lane_costs: []const Cost = &.{}, // the same for the lane planner (Model.lane_costs) tree_nodes: u32 = 0, // hold() without lanes grows its own tree of up to `depth` nodes, at most this many (tree, chances) + chances: bool = false, // chances() gives each held chain draft's chance: verify rows go by them (Model.granted_depth) + confidence_floor: ?f64 = null, // with chances: every round drafts the deepest chain and verifies drafts this likely + slack: u32 = 0, // with chances: drafts a hold offers past the stream's last grant (0: the depth rule's own) + copy_first: bool = false, // a hold costs a pass of its own: none is asked for a stream with a copy ahead + hold_costs: []const @import("hold_price.zig").HoldCost = &.{}, // that pass's ms by blocks and depth (timed at load) }; /// Target rows [start, start + features.rows) and the token after each; `rows`, when given, picks which to absorb. diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index 5d357c646d..3bf6c297da 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -16,6 +16,7 @@ const lone_costs = @import("lone_costs.zig"); const open = @import("open.zig"); const handover = @import("handover.zig"); const trim = @import("trim.zig"); +const pipelined = @import("pipelined.zig"); const Stream = sm.Stream; const Feed = be.Feed; const LogRow = @import("logprob.zig").Row; @@ -24,7 +25,7 @@ const f = ev.f; const str = trail.str; const int = trail.int; const Outcome = struct { got: []const u32, path: []const u32, cut: ?usize, follow: []const u32 }; -const Result = struct { rows: u32, keep: u32, got: []const u32 }; +pub const Result = struct { rows: u32, keep: u32, got: []const u32 }; const Unread = struct { event: usize, handle: u64 }; pub const Engine = struct { const pieces_ = @import("pieces.zig"); @@ -70,6 +71,7 @@ pub const Engine = struct { .draft_step_ms = cfg.draft_step_ms, .plain_guard = cfg.plain_guard, .node_probabilities = cfg.node_probabilities, + .granted_depth = cfg.granted_depth, .confidence_floor = cfg.draft_confidence, .batch_rows = cfg.batch_rows, .batched_heads = cfg.draft_streams, @@ -124,7 +126,7 @@ pub const Engine = struct { if (live.items.len > 1 and e.cfg.family_streams) { // a shared round is synchronous: a stream that ran a step ahead lands its queued token first for (live.items) |s| { - if (s.inflight != null) try e.landInflight(s); + if (s.inflight != null) try pipelined.land(e, s); } var kept: usize = 0; for (live.items) |s| { @@ -147,7 +149,7 @@ pub const Engine = struct { for (live.items) |s| { // logprobs and grammars take verify windows: rows' logits are read, or bits made, before a forward const windowed = (e.cfg.family_mtp and s.drafts) or !e.cfg.pipelined or s.logprobs != null or s.grammar != null; - const r = if (windowed) try e.familyRound(s, null) else try e.pipelinedRound(s); + const r = if (windowed) try e.familyRound(s, null) else try pipelined.round(e, s); s.min_rows = if (s.min_rows == 0) r.rows else @min(s.min_rows, r.rows); } } @@ -195,7 +197,7 @@ pub const Engine = struct { } /// One stream's round: verify pending and drafted tokens together, keep drafts to the first mismatch. - fn familyRound(e: *Engine, s: *Stream, copied: ?win.Copy) !Result { + pub fn familyRound(e: *Engine, s: *Stream, copied: ?win.Copy) !Result { e.clock.start(); const priced = s.priced; s.priced = null; @@ -490,64 +492,6 @@ pub const Engine = struct { } } - /// One token, the next forward queued first; a copied continuation ahead switches to verify windows. - fn pipelinedRound(e: *Engine, s: *Stream) !Result { - const before = s.mode; - // an adopted stream holds neither a mode nor a queued step: it starts where a drain step leaves one - const mode: sm.Mode = s.mode orelse if (s.inflight == null) .verify else .pipe; - var result: Result = .{ .rows = 1, .keep = 1, .got = &.{} }; - if (mode == .verify or mode == .exit) { - const verifying = mode == .verify and e.cfg.family_width >= 2; - const proposal: win.Copy = if (verifying) try win.copyProposal(e, s) else .{}; - if (proposal.tokens.len > 0) { - const copied = try e.arena.allocator().dupe(u32, proposal.tokens); - result = try e.familyRound(s, .{ .tokens = copied, .recalled = proposal.recalled }); - if (result.keep == 1 or s.force.items.len > 0) s.mode = .exit; - } else { - // nothing to copy: back to steps queued ahead (this one lands next round) - s.mode = .pipe; - try e.queueNext(s, .{ .value = s.pending.? }); - } - try trail.pipe(e, s, before, result.got); - return result; - } - var current: Feed = .{ .handle = s.inflight.? }; - s.inflight = null; - if (try e.forcedNext(s)) |t| current = .{ .value = t }; - var token: u32 = undefined; - if (mode == .drain) { - token = try e.readFeed(current); // the last queued step: no new one - s.mode = .verify; - } else { - try e.queueNext(s, current); // the GPU starts the next step first - token = try e.readFeed(current); - } - s.rounds += 1; - const landed = try s.commit(e.gpa, &.{token}, &.{}); - s.pending = token; - if (s.recall) |r| r.noteHead(1); // a step lands one token: the bar a copy must beat - if (e.cfg.root_copies) s.root_copy_miss = false; - const piping = e.cfg.family_width >= 2 and (s.mode orelse .pipe) == .pipe and !s.finished and s.drafts; - if (piping and (try win.copyProposal(e, s)).tokens.len > 0) - s.mode = .drain; // a copy window is ahead: land the queued step - result.got = try e.arena.allocator().dupe(u32, s.emitted()[s.emitted().len - landed ..]); - try trail.pipe(e, s, before, result.got); - return result; - } - - /// Land the queued token without queuing another (a shared round is synchronous). - fn landInflight(e: *Engine, s: *Stream) !void { - var current: Feed = .{ .handle = s.inflight.? }; - s.inflight = null; - if (try e.forcedNext(s)) |t| current = .{ .value = t }; - const token = try e.readFeed(current); - s.rounds += 1; - const landed = try s.commit(e.gpa, &.{token}, &.{}); - s.pending = token; - s.mode = .verify; - try trail.event(e, &.{ f("ev", str("land")), f("stream", str(s.id)), f("got", .{ .u32s = s.emitted()[s.emitted().len - landed ..] }) }); - } - /// The first token's draw: among the grammar's allowed tokens when the stream has one that constrains it. pub fn drawFirst(e: *Engine, s: *Stream, position: u64) !u64 { const g = s.grammar orelse return e.backend.first(s, position); diff --git a/zig/src/core/lanes/hold_price.zig b/zig/src/core/lanes/hold_price.zig new file mode 100644 index 0000000000..c3eeb0208a --- /dev/null +++ b/zig/src/core/lanes/hold_price.zig @@ -0,0 +1,84 @@ +//! A drafter's hold pass priced from its timed passes: by the blocks it holds and their depth. +const std = @import("std"); + +/// A drafter's hold pass of `streams` blocks, each `depth` drafts deep, in ms (timed at load). +pub const HoldCost = struct { streams: u32, depth: u32, ms: f64 }; + +/// A hold pass's ms: linear in depth at each timed stream count, then in streams (past the widest, by its slope). +pub fn holdMs(costs: []const HoldCost, streams: usize, depth: i64) f64 { + var below: ?u32 = null; + var above: ?u32 = null; + for (costs) |c| { + if (c.streams <= streams and (below == null or c.streams > below.?)) below = c.streams; + if (c.streams >= streams and (above == null or c.streams < above.?)) above = c.streams; + } + const lo = below orelse above.?; + const hi = above orelse lo; + const at_lo = atDepth(costs, lo, depth); + if (hi == lo) { + if (above != null) return at_lo; + // past the widest timed count: the slope from the count below it + var under: ?u32 = null; + for (costs) |c| if (c.streams < lo and (under == null or c.streams > under.?)) { + under = c.streams; + }; + const u = under orelse return at_lo * @as(f64, @floatFromInt(streams)) / @as(f64, @floatFromInt(lo)); + const slope = (at_lo - atDepth(costs, u, depth)) / @as(f64, @floatFromInt(lo - u)); + return at_lo + slope * @as(f64, @floatFromInt(streams - lo)); + } + const t = @as(f64, @floatFromInt(streams - lo)) / @as(f64, @floatFromInt(hi - lo)); + return at_lo + t * (atDepth(costs, hi, depth) - at_lo); +} + +/// Interpolate neighboring depths, or extrapolate the nearest end segment (one alone: as timed). +fn atDepth(costs: []const HoldCost, streams: u32, depth: i64) f64 { + var a: ?HoldCost = null; + var b: ?HoldCost = null; + for (costs) |c| { + if (c.streams != streams) continue; + if (c.depth == depth) return c.ms; + if (c.depth < depth and (a == null or c.depth > a.?.depth)) a = c; + if (c.depth > depth and (b == null or c.depth < b.?.depth)) b = c; + } + if (a == null) { + a = b.?; + b = null; + for (costs) |c| if (c.streams == streams and c.depth > a.?.depth and (b == null or c.depth < b.?.depth)) { + b = c; + }; + if (b == null) return a.?.ms; + } else if (b == null) { + b = a.?; + a = null; + for (costs) |c| if (c.streams == streams and c.depth < b.?.depth and (a == null or c.depth > a.?.depth)) { + a = c; + }; + if (a == null) return b.?.ms; + } + const x = a.?; + const y = b.?; + const t = (@as(f64, @floatFromInt(depth)) - @as(f64, @floatFromInt(x.depth))) / @as(f64, @floatFromInt(y.depth - x.depth)); + return x.ms + t * (y.ms - x.ms); +} + +test "hold prices retain measured kernel transitions and use the adjacent depth segment" { + const t = [_]HoldCost{ + .{ .streams = 8, .depth = 4, .ms = 18 }, + .{ .streams = 8, .depth = 1, .ms = 10 }, + .{ .streams = 8, .depth = 2, .ms = 17 }, + }; + try std.testing.expectEqual(@as(f64, 17), holdMs(&t, 8, 2)); + try std.testing.expectEqual(@as(f64, 17.5), holdMs(&t, 8, 3)); + try std.testing.expectEqual(@as(f64, 18.5), holdMs(&t, 8, 5)); + try std.testing.expectEqual(@as(f64, 3), holdMs(&t, 8, 0)); +} + +test "a hold's ms between and past the timed passes" { + const t = [_]HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1.0 }, .{ .streams = 1, .depth = 4, .ms = 2.5 }, .{ .streams = 4, .depth = 1, .ms = 3.0 }, .{ .streams = 4, .depth = 4, .ms = 4.5 }, .{ .streams = 8, .depth = 1, .ms = 4.0 } }; + try std.testing.expectApproxEqAbs(@as(f64, 1.0), holdMs(&t, 1, 1), 1e-12); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), holdMs(&t, 1, 3), 1e-12); // linear in depth + try std.testing.expectApproxEqAbs(@as(f64, 3.0), holdMs(&t, 1, 5), 1e-12); // and past the deepest + try std.testing.expectApproxEqAbs(@as(f64, 1.0 + 2.0 / 3.0), holdMs(&t, 2, 1), 1e-12); // linear in streams + try std.testing.expectApproxEqAbs(@as(f64, 4.0), holdMs(&t, 8, 3), 1e-12); // one depth timed: as timed + try std.testing.expectApproxEqAbs(@as(f64, 5.0), holdMs(&t, 12, 1), 1e-12); // past the widest: the slope below it +} diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig index 280b16ef10..cc0271d636 100644 --- a/zig/src/core/lanes/lanes.zig +++ b/zig/src/core/lanes/lanes.zig @@ -18,6 +18,7 @@ pub const cost_rule = @import("cost_rule.zig"); pub const gpu_rule = @import("gpu_rule.zig"); pub const gpu_full = @import("gpu_full.zig"); pub const depth = @import("depth.zig"); +pub const hold_price = @import("hold_price.zig"); pub const mtp_policy = @import("mtp_policy.zig"); pub const trim = @import("trim.zig"); pub const config = @import("config.zig"); @@ -57,6 +58,8 @@ test { _ = @import("recorder_test.zig"); _ = @import("gpu_full_test.zig"); _ = @import("drafted_test.zig"); + _ = @import("drafted_chances_test.zig"); + _ = @import("drafted_copy_test.zig"); _ = @import("root_copy_test.zig"); _ = @import("drafted_grammar_test.zig"); _ = @import("windows_grammar_test.zig"); diff --git a/zig/src/core/lanes/pipelined.zig b/zig/src/core/lanes/pipelined.zig new file mode 100644 index 0000000000..99de33304b --- /dev/null +++ b/zig/src/core/lanes/pipelined.zig @@ -0,0 +1,70 @@ +//! A lone stream's pipelined steps: the next forward queued before the last one's token is read. +const be = @import("backend.zig"); +const sm = @import("stream.zig"); +const ev = @import("events.zig"); +const win = @import("windows.zig"); +const trail = @import("trail.zig"); +const Engine = @import("engine.zig").Engine; +const Result = @import("engine.zig").Result; +const Stream = sm.Stream; +const Feed = be.Feed; +const f = ev.f; +const str = trail.str; + +/// One token, the next forward queued first; a copied continuation ahead switches to verify windows. +pub fn round(e: *Engine, s: *Stream) !Result { + const before = s.mode; + // an adopted stream holds neither a mode nor a queued step: it starts where a drain step leaves one + const mode: sm.Mode = s.mode orelse if (s.inflight == null) .verify else .pipe; + var result: Result = .{ .rows = 1, .keep = 1, .got = &.{} }; + if (mode == .verify or mode == .exit) { + const verifying = mode == .verify and e.cfg.family_width >= 2; + const proposal: win.Copy = if (verifying) try win.copyProposal(e, s) else .{}; + if (proposal.tokens.len > 0) { + const copied = try e.arena.allocator().dupe(u32, proposal.tokens); + result = try e.familyRound(s, .{ .tokens = copied, .recalled = proposal.recalled }); + if (result.keep == 1 or s.force.items.len > 0) s.mode = .exit; + } else { + // nothing to copy: back to steps queued ahead (this one lands next round) + s.mode = .pipe; + try e.queueNext(s, .{ .value = s.pending.? }); + } + try trail.pipe(e, s, before, result.got); + return result; + } + var current: Feed = .{ .handle = s.inflight.? }; + s.inflight = null; + if (try e.forcedNext(s)) |t| current = .{ .value = t }; + var token: u32 = undefined; + if (mode == .drain) { + token = try e.readFeed(current); // the last queued step: no new one + s.mode = .verify; + } else { + try e.queueNext(s, current); // the GPU starts the next step first + token = try e.readFeed(current); + } + s.rounds += 1; + const landed = try s.commit(e.gpa, &.{token}, &.{}); + s.pending = token; + if (s.recall) |r| r.noteHead(1); // a step lands one token: the bar a copy must beat + if (e.cfg.root_copies) s.root_copy_miss = false; + const piping = e.cfg.family_width >= 2 and (s.mode orelse .pipe) == .pipe and !s.finished and s.drafts; + if (piping and (try win.copyProposal(e, s)).tokens.len > 0) + s.mode = .drain; // a copy window is ahead: land the queued step + result.got = try e.arena.allocator().dupe(u32, s.emitted()[s.emitted().len - landed ..]); + try trail.pipe(e, s, before, result.got); + return result; +} + +/// Land the queued token without queuing another (a shared round is synchronous). +pub fn land(e: *Engine, s: *Stream) !void { + var current: Feed = .{ .handle = s.inflight.? }; + s.inflight = null; + if (try e.forcedNext(s)) |t| current = .{ .value = t }; + const token = try e.readFeed(current); + s.rounds += 1; + const landed = try s.commit(e.gpa, &.{token}, &.{}); + s.pending = token; + s.mode = .verify; + try trail.event(e, &.{ f("ev", str("land")), f("stream", str(s.id)), f("got", .{ .u32s = s.emitted()[s.emitted().len - landed ..] }) }); +} diff --git a/zig/src/core/lanes/stream.zig b/zig/src/core/lanes/stream.zig index c3e22e7869..ff85c8a3e7 100644 --- a/zig/src/core/lanes/stream.zig +++ b/zig/src/core/lanes/stream.zig @@ -234,6 +234,8 @@ pub const Stream = struct { served: ?u64 = null, proposal_depth: ?u32 = null, // null: legacy; otherwise the last routed request, zero until it succeeds granted: ?u32 = null, + grant_short: bool = false, // the last grant verified fewer drafts than the stream held + grant_gain: ?f64 = null, // the drafts its last grant expected to keep (the chances it was priced by) copy_width: ?u32 = null, pub fn init(gpa: Allocator, spec: Spec) !Stream { @@ -302,6 +304,8 @@ pub const Stream = struct { s.inflight = null; s.served = null; s.granted = null; + s.grant_short = false; + s.grant_gain = null; s.copy_width = null; s.proposal_depth = null; } diff --git a/zig/src/core/lanes/windows.zig b/zig/src/core/lanes/windows.zig index fd7ffec95d..c9e8e8ea2b 100644 --- a/zig/src/core/lanes/windows.zig +++ b/zig/src/core/lanes/windows.zig @@ -226,7 +226,7 @@ pub fn probe(e: *Engine) depth.Probe { } pub fn who(s: *Stream) depth.Who { - return .{ .state = &s.depth, .draft_room = s.draftRoom(), .finished = s.finished, .forced = s.force.items.len > 0, .granted = s.granted, .stream = s, .sampled = s.sampling != null }; + return .{ .state = &s.depth, .draft_room = s.draftRoom(), .finished = s.finished, .forced = s.force.items.len > 0, .granted = s.granted, .grant_short = s.grant_short, .gain = s.grant_gain, .stream = s, .sampled = s.sampling != null }; } /// Whether any of a round's streams draws keyed samples (its rows carry their host work). @@ -282,8 +282,16 @@ pub fn allocate(e: *Engine, plans: []Plan) !void { const ms = own.get(1 + @as(i64, counts[0])); try trail.event(e, &.{ f("ev", trail.str("lone-price")), f("stream", trail.str(plans[0].stream.id)), f("rows", trail.int(counts[0] + 1)), f("expected", ev.bits(expected)), f("ms", if (ms) |value| ev.bits(value) else .null), f("probe", .{ .bool = trial != null }) }); } - for (plans, counts) |*p, c| { - if (p.kind == .head) p.stream.granted = c; + for (plans, counts, probs) |*p, c, pr| { + if (p.kind == .head) { + p.stream.granted = c; + if (e.cfg.granted_depth) { + p.stream.grant_short = c < p.count(); + var gain: f64 = 0; + for (pr[0..@min(c, pr.len)]) |chance| gain += chance; + p.stream.grant_gain = gain; + } + } if (p.kind == .forced or p.count() == 0 or c == p.count()) continue; try cut.prefix(a, p, c); } @@ -393,7 +401,7 @@ fn planChances(e: *Engine, p: Plan, n: usize) ![]const f64 { if (!p.context_copy or p.branch_from == 0 or p.parents == null or n < p.branch_from) return nodeChances(e, p.stream, n); const drafts = p.branch_from - 1; const out = try e.arena.allocator().alloc(f64, n); - if (drafts > 0) @memcpy(out[0..drafts], try nodeChances(e, p.stream, drafts)); + if (drafts > 0) try nodeChancesInto(e, p.stream, out[0..drafts]); const rate = copyRate(e, p.stream, p.copy_trusted); for (out[drafts..], p.parents.?[drafts..n]) |*o, parent| o.* = rate * (if (parent >= 0) out[@intCast(parent)] else 1); return out; @@ -460,32 +468,36 @@ test "a copy is trusted after a long match, or out of the prompt early in the re /// The head's chance for each held draft, else chain chances from the stream's per-depth acceptance. pub fn nodeChances(e: *Engine, s: *Stream, n: usize) ![]const f64 { - const a = e.arena.allocator(); + const out = try e.arena.allocator().alloc(f64, n); + try nodeChancesInto(e, s, out); + return out; +} + +fn nodeChancesInto(e: *Engine, s: *Stream, out: []f64) !void { if (e.cfg.node_probabilities) { if (e.backend.vtable.probabilities) |chances| { - const out = try a.alloc(f64, n); - if (try chances(e.backend.ptr, s, out)) return out; + if (try chances(e.backend.ptr, s, out)) return; } } if (e.cfg.draft_confidence != null) return error.NoDraftProbabilities; - return alloc.chainProbabilities(a, try e.rule.rates(who(s)), n); + alloc.chainProbabilitiesInto(try e.rule.rates(who(s)), out); } /// Structured output: the rows an accepted path can use, each row's bits, and the grammar's drafts after a lone chain. pub fn constrain(e: *Engine, p: *Plan) !void { const g = p.stream.grammar orelse return; const a = e.arena.allocator(); - if (p.held > 0) { - // the held drafts (a chain or a tree, in window order) come back as host tokens, grafts after them + const held_count = p.held; + const rows = try a.alloc(u32, 1 + held_count + p.tokens.len); + rows[0] = p.stream.pending.?; + @memcpy(rows[1 + held_count ..], p.tokens); + if (held_count > 0) { + // Read held drafts directly into their final rows, followed by any grafts. const read = e.backend.vtable.held orelse return error.HeldDraftsUnreadable; - const held = try a.alloc(u32, p.held); - try read(e.backend.ptr, p.stream, held); - p.tokens = try std.mem.concat(a, u32, &.{ held, p.tokens }); + try read(e.backend.ptr, p.stream, rows[1 .. 1 + held_count]); + p.tokens = rows[1..]; p.held = 0; } - const rows = try a.alloc(u32, 1 + p.tokens.len); - rows[0] = p.stream.pending.?; - @memcpy(rows[1..], p.tokens); const grows = p.stream.drafts and p.kind != .forced and e.alone; const room = if (grows) @max(rows.len, e.cfg.family_width) else rows.len; const w = try g.window(a, rows, try accept.rowParents(a, rows.len, p.parents), p.kind == .forced, room); @@ -540,3 +552,7 @@ pub fn tokens(e: *Engine, w: be.Window, o: be.Verified) ![]u32 { @memcpy(out[1 + w.held ..], w.tokens); return out; } + +test "declined and grafted probabilities need only their final round buffer" { + try @import("windows_memory_test.zig").check(nodeChances, planChances); +} diff --git a/zig/src/core/lanes/windows_grammar_test.zig b/zig/src/core/lanes/windows_grammar_test.zig index fee374c63b..c90ce91b61 100644 --- a/zig/src/core/lanes/windows_grammar_test.zig +++ b/zig/src/core/lanes/windows_grammar_test.zig @@ -93,3 +93,38 @@ test "a tree shaped as a chain takes the grammar's own drafts as chain rows, and try std.testing.expectEqual(@as(usize, 0), g.at); } } + +test "held drafts and grafts keep row order through grammar trimming" { + var cfg = try et.model(); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var backend = target.backend(); + var vtable = backend.vtable.*; + vtable.held = struct { + fn read(_: *anyopaque, _: *sm.Stream, out: []u32) !void { + try std.testing.expectEqual(@as(usize, 2), out.len); + @memcpy(out, &[_]u32{ 5, 7 }); + } + }.read; + backend.vtable = &vtable; + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, backend, clock.clock()); + defer engine.deinit(); + for ([_]u32{ 9, 4 }) |graft| { + var g: Spell = .{ .want = &.{ 5, 7, 9 } }; + var k: grammar.Constraint = .init(g.rules(), (fake.vocab + 31) / 32, null); + var s = try sm.Stream.init(gpa, .{ .id = "held", .prompt = &.{ 1, 2 }, .max_new = 8, .drafts = false, .grammar = &k }); + defer s.deinit(gpa); + s.pending = 3; + const tokens = [_]u32{graft}; + var p: win.Plan = .{ .stream = &s, .position = 2, .kind = .head, .held = 2, .tokens = &tokens }; + try win.constrain(&engine, &p); + try std.testing.expectEqual(@as(u32, 0), p.held); + const expected: []const u32 = &.{ 5, 7, 9 }; + try std.testing.expectEqual(@as(u32, if (graft == 9) 0 else 3), p.grammar_from); + try std.testing.expectEqualSlices(u32, expected, p.tokens); + const w = try win.build(&engine, p, false); + try std.testing.expectEqualSlices(u32, expected, w.tokens); + } +} diff --git a/zig/src/core/lanes/windows_memory_test.zig b/zig/src/core/lanes/windows_memory_test.zig new file mode 100644 index 0000000000..db08abf447 --- /dev/null +++ b/zig/src/core/lanes/windows_memory_test.zig @@ -0,0 +1,50 @@ +//! Probability planning must fit its final output into a bounded round arena. +const std = @import("std"); +const Engine = @import("engine.zig").Engine; +const Stream = @import("stream.zig").Stream; +const win = @import("windows.zig"); + +pub fn check(comptime node: anytype, comptime hybrid: anytype) !void { + const a = std.testing.allocator; + var cfg = try @import("config.zig").Config.init(a, .{ .exact_width = 16, .drafts = 3, .mtp = true, .speculate = true, .draft_probabilities = true, .draft_prior = &.{ 0.8, 0.5 } }, 16, 3); + defer cfg.deinit(a); + var target: @import("fake.zig").Fake = .{ .gpa = a }; + defer target.deinit(); + var clock: @import("fake.zig").FixedClock = .{}; + var backend = target.backend(); + var vtable = backend.vtable.*; + vtable.probabilities = struct { + fn decline(_: *anyopaque, _: *Stream, out: []f64) !bool { + @memset(out, 99); + return false; + } + }.decline; + backend.vtable = &vtable; + var limited = std.testing.FailingAllocator.init(a, .{}); + var e = Engine.init(a, &cfg, backend, clock.clock()); + defer e.deinit(); + var s = try Stream.init(a, .{ .id = "bounded", .prompt = &.{ 1, 2 }, .max_new = 64 }); + defer s.deinit(a); + _ = try e.rule.rates(win.who(&s)); + // The round gets only enough arena space for its final probability buffer. + e.arena.deinit(); + e.arena = std.heap.ArenaAllocator.init(limited.allocator()); + _ = try e.arena.allocator().alloc(f64, 32); + const capacity = e.arena.queryCapacity(); + limited.fail_index = limited.alloc_index; + limited.resize_fail_index = limited.resize_index; + for (0..2) |mode| { + try std.testing.expect(e.arena.reset(.retain_capacity)); + _ = try e.arena.allocator().alloc(u8, capacity - 16 * @sizeOf(f64)); + var parents: [16]i32 = undefined; + @memset(&parents, 0); + parents[0] = -1; + const p: win.Plan = .{ .stream = &s, .position = 2, .kind = .head, .tokens = &.{}, .parents = &parents, .branch_from = 4, .context_copy = true, .copy_trusted = false }; + const out = if (mode == 0) try node(&e, &s, 16) else try hybrid(&e, p, 16); + try std.testing.expectApproxEqAbs(@as(f64, 0.8), out[0], 1e-12); + try std.testing.expectApproxEqAbs(@as(f64, 0.4), out[1], 1e-12); + try std.testing.expectApproxEqAbs(@as(f64, 0.2), out[2], 1e-12); + if (mode == 1) try std.testing.expectApproxEqAbs(@as(f64, 0.064), out[3], 1e-12); + } + try std.testing.expect(!limited.has_induced_failure); +} diff --git a/zig/src/core/tile_qmm.zig b/zig/src/core/tile_qmm.zig index 030bd097e1..dbc0da8b68 100644 --- a/zig/src/core/tile_qmm.zig +++ b/zig/src/core/tile_qmm.zig @@ -167,10 +167,26 @@ pub fn encodeTwin( y: mtl.Buffer, y_off: usize, rows: usize, +) !void { + // the window kernel only for matrices of window_n outputs or more + return encodeTwinFrom(e, p, m, x, x_off, y, y_off, rows, if (m.n >= window_n) window_rows else twin_rows + 1); +} + +/// encodeTwin with the window kernel from `from` rows on (the scalar twin below them), for a caller's measured cut. +pub fn encodeTwinFrom( + e: mtl.ComputeEncoder, + p: *const Pipelines, + m: Matrix, + x: mtl.Buffer, + x_off: usize, + y: mtl.Buffer, + y_off: usize, + rows: usize, + from: usize, ) !void { const i = try width(m); if (rows == 0 or rows > twin_rows) return error.NoRows; - const window = rows >= window_rows and m.n >= window_n; + const window = rows >= from; e.setPipeline(if (window) p.rows[i] else if (rows > 1) p.twin_few[i] else p.twin[i]); bind(e, m, .{ .k = @intCast(m.k), .n = @intCast(m.n), .r = @intCast(rows) }, x, x_off, y, y_off); if (window) return e.dispatchGroups(.{ .width = m.n / window_columns }, .{ .width = 64 }); diff --git a/zig/src/families/lfm2_moe/backend.zig b/zig/src/families/lfm2_moe/backend.zig index d713c9ec24..2e5130db84 100644 --- a/zig/src/families/lfm2_moe/backend.zig +++ b/zig/src/families/lfm2_moe/backend.zig @@ -10,6 +10,14 @@ const snapshot = @import("snapshot.zig"); const Model = @import("model.zig").Model; const be = lanes.backend; +/// Where a drafter holds a stream's drafts on the device, for verify windows that take them there (Window.held). +pub const HeldSource = struct { + ptr: *anyopaque, + get: *const fn (ptr: *anyopaque, s: *lanes.Stream) ?Device, + host: *const fn (ptr: *anyopaque, s: *lanes.Stream, out: []u32) anyerror!void, // where `get` gives none +}; +pub const Device = struct { src: mtl.Buffer, off: usize, stride: u32, wait: fwd.Wait }; + pub const Options = struct { capacity: usize, chunk: usize = 128, streams: u32 = 8 }; pub const Metal = struct { @@ -23,6 +31,13 @@ pub const Metal = struct { /// Forward ms by window width, timed at load: the lane core weighs drafts against them. costs: [st.window_rows]lanes.config.Cost = undefined, timed: usize = 0, + /// Shared-row increments past the qualified window, anchored at that window's measured cost. + shared: [st.batch_rows - st.window_rows]lanes.config.Cost = undefined, + shared_timed: usize = 0, + /// Forwards run so far: a stream's tapped rows in the scratch are current while their gen matches. + gen: u64 = 0, + /// A drafter whose drafts verify windows read on the device (null: windows carry host tokens only). + held_source: ?HeldSource = null, pub fn init(gpa: std.mem.Allocator, model: *Model, options: Options) !*Metal { const out = try gpa.create(Metal); @@ -54,7 +69,9 @@ pub const Metal = struct { } pub fn facts(self: *const Metal) lanes.Model { - return factsFor(self.width, self.options.streams, self.costs[0..self.timed]); + var out = factsFor(self.width, self.options.streams, self.costs[0..self.timed]); + out.shared_costs = self.shared[0..self.shared_timed]; + return out; } /// Each window width's forward ms on a 256-token history (the fastest of three, after the GPU's clocks ramp up). @@ -84,11 +101,37 @@ pub const Metal = struct { cost.* = .{ .width = @intCast(w), .ms = best }; } std.log.info("LFM2 window ms by width: 1 {d:.1}, 2 {d:.1}, 4 {d:.1}, 8 {d:.1}, 16 {d:.1}", .{ self.costs[0].ms, self.costs[1].ms, self.costs[3].ms, self.costs[7].ms, self.costs[15].ms }); + // Two qualified windows measure shared-row increments against the same shared shape at 16 rows. + var other = try st.Cache.init(self.gpa, self.model.device, &self.model.config.attention, 512); + defer other.deinit(); + try fwd.run(self.model, &self.scratch, &.{.{ .cache = &other, .rows = 256 }}, &ids, false, .last); + var shared_base: f64 = 0; + for (st.window_rows..st.batch_rows + 1) |rows| { + const first = @min(st.window_rows, rows - 1); + var best: f64 = std.math.floatMax(f64); + for (0..4) |trial| { + var t0: platform.c.timespec = undefined; + var t1: platform.c.timespec = undefined; + _ = platform.c.clock_gettime(platform.c.CLOCK.MONOTONIC, &t0); + try fwd.run(self.model, &self.scratch, &.{ .{ .cache = &cache, .rows = first }, .{ .cache = &other, .rows = rows - first } }, ids[0..rows], true, .all); + _ = platform.c.clock_gettime(platform.c.CLOCK.MONOTONIC, &t1); + const ms = @as(f64, @floatFromInt(t1.sec - t0.sec)) * 1e3 + @as(f64, @floatFromInt(t1.nsec - t0.nsec)) / 1e6; + cache.len = base; + other.len = base; + if (trial > 0) best = @min(best, ms); + } + if (rows == st.window_rows) { + shared_base = best; + } else self.shared[rows - st.window_rows - 1] = .{ .width = @intCast(rows), .ms = self.costs[st.window_rows - 1].ms + best - shared_base }; + } + self.shared_timed = self.shared.len; + std.log.info("LFM2 shared calibration: window16 {d:.2} ms, shared16 {d:.2} ms", .{ self.costs[st.window_rows - 1].ms, shared_base }); + std.log.info("LFM2 shared marginal prices by rows: 17 {d:.1}, 24 {d:.1}, 32 {d:.1}", .{ self.shared[0].ms, self.shared[7].ms, self.shared[15].ms }); return self.costs.len; } pub fn backend(self: *Metal) be.Backend { - return .{ .ptr = self, .vtable = &.{ .prefill = prefillFn, .first = firstFn, .first_masked = firstMaskedFn, .queue = queueFn, .read = readFn, .verify = verifyFn, .keep = keepFn, .draft = draftFn, .release = releaseFn } }; + return .{ .ptr = self, .vtable = &.{ .prefill = prefillFn, .prepare_features = prepareFn, .features = featuresFn, .prefill_features = prefillFeaturesFn, .first = firstFn, .first_masked = firstMaskedFn, .queue = queueFn, .read = readFn, .verify = verifyFn, .keep = keepFn, .draft = draftFn, .release = releaseFn } }; } fn cast(ptr: *anyopaque) *Metal { @@ -107,8 +150,65 @@ pub const Metal = struct { return cache; } + /// One forward on the scratch; a stream's tapped rows from earlier forwards are stale after it. + fn run(self: *Metal, segments: []const fwd.Segment, tokens: []const u32, record: bool, head: fwd.Head) !void { + self.gen += 1; + try fwd.run(self.model, &self.scratch, segments, tokens, record, head); + } + fn prefillFn(ptr: *anyopaque, stream: *lanes.Stream) !void { + return cast(ptr).prefillWith(stream, null); + } + + /// The prompt pass, `sink` taking each chunk's tapped rows before the chunk's end is kept for reuse. + fn prefillFeaturesFn(ptr: *anyopaque, stream: *lanes.Stream, layers: []const u32, sink: be.FeatureSink) !void { + const self = cast(ptr); + const t = self.scratch.taps orelse return error.UnpreparedLfmTaps; + if (layers.len != t.count) return error.UnpreparedLfmTaps; + return self.prefillWith(stream, sink); + } + + /// A tapped row's bytes: every tap's hidden row, bf16. + fn tapBytes(self: *const Metal) usize { + return if (self.scratch.taps) |t| t.count * c.hidden * 2 else 0; + } + + /// The drafter's layers: every forward from now on keeps their outputs for each row. + fn prepareFn(ptr: *anyopaque, layers: []const u32) !void { const self = cast(ptr); + if (layers.len == 0 or layers.len > st.max_taps) return error.BadLfmTaps; + for (layers) |l| if (l >= c.layers) return error.BadLfmTaps; + if (self.scratch.taps) |t| return if (std.mem.eql(u32, t.layers[0..t.count], layers)) {} else error.BadLfmTaps; + var t: st.Taps = .{ .count = @intCast(layers.len), .buf = undefined }; + @memcpy(t.layers[0..layers.len], layers); + t.buf = try self.scratch.memory.alloc(self.scratch.rows * layers.len * c.hidden * 2); + self.scratch.taps = t; + } + + /// Cache rows [start, start + count) of the stream's latest tapped rows, on the device. + fn featuresFn(ptr: *anyopaque, stream: *lanes.Stream, layers: []const u32, start: u64, count: u32) !be.Features { + const self = cast(ptr); + const t = self.scratch.taps orelse return error.UnpreparedLfmTaps; + if (layers.len != t.count) return error.UnpreparedLfmTaps; + const cache = try self.cacheOf(stream); + const held = cache.tap orelse return error.LfmTapsNotHeld; + if (count == 0 or start < held.start or start + count > held.start + held.rows) return error.LfmTapsNotHeld; + const width = self.tapBytes(); + const buf = if (held.gen == 0) cache.last_tap.? else if (held.gen == self.gen) t.buf else return error.StaleLfmTaps; + const row = held.row + @as(usize, @intCast(start - held.start)); + return .{ .buffer = @intFromPtr(buf.id), .offset = row * width, .rows = count, .row_bytes = @intCast(width), .space = .device, .dtype = .bf16 }; + } + + /// The prompt's last row's taps, copied beside the stream's caches for its first draft (gen 0: kept there). + fn keepLastTap(self: *Metal, cache: *st.Cache, row: usize, position: usize) !void { + const width = self.tapBytes(); + if (width == 0) return; + if (cache.last_tap == null) cache.last_tap = try cache.memory.alloc(width); + @memcpy(cache.last_tap.?.contents()[0..width], (self.scratch.taps.?.buf.contents() + row * width)[0..width]); + cache.tap = .{ .start = position, .row = 0, .rows = 1, .gen = 0 }; + } + + fn prefillWith(self: *Metal, stream: *lanes.Stream, sink: ?be.FeatureSink) !void { const ids = stream.prompt(); if (ids.len == 0 or ids.len + stream.max_new > self.options.capacity or self.caches.contains(stream)) return error.InvalidLfmPrompt; const capacity = @min(self.options.capacity, ids.len + stream.max_new + st.window_rows); @@ -121,7 +221,9 @@ pub const Metal = struct { stream.cached = 0; if (stream.reuse.saved) |saved| { // a kept state of this prompt's prefix: the pass starts there const snap: *snapshot.Snap = @ptrCast(@alignCast(saved)); - if (snap.at < ids.len) if (snapshot.restore(cache, snap)) |_| { + // a drafting stream resumes only a state kept with its drafter's context + const drafter_less = sink != null and snap.draft == null; + if (snap.at < ids.len and !drafter_less) if (snapshot.restore(cache, snap)) |_| { at = snap.at; } else |_| {}; stream.cached = @intCast(at); @@ -136,7 +238,13 @@ pub const Metal = struct { }; while (mark < stream.reuse.marks.len and stream.reuse.marks[mark] <= at) mark += 1; if (mark < stream.reuse.marks.len) end = @min(end, stream.reuse.marks[mark]); - try fwd.run(self.model, &self.scratch, &.{.{ .cache = cache, .rows = end - at }}, ids[at..end], false, .last); + try self.run(&.{.{ .cache = cache, .rows = end - at }}, ids[at..end], false, .last); + if (sink) |k| { + const width = self.tapBytes(); + const f: be.Features = .{ .buffer = @intFromPtr(self.scratch.taps.?.buf.id), .rows = @intCast(end - at), .row_bytes = @intCast(width), .space = .device, .dtype = .bf16 }; + try k.take(k.ptr, stream, f, at); + if (end == ids.len) try self.keepLastTap(cache, end - 1 - at, end - 1); + } at = end; if (stream.reuse.hook) |hk| if (std.mem.indexOfScalar(u32, stream.reuse.marks, @intCast(at)) != null) hk.at(hk.ptr, stream, @intCast(at)); @@ -165,7 +273,8 @@ pub const Metal = struct { .value => |v| v, .handle => |h| @intCast(h), }; - try fwd.run(self.model, &self.scratch, &.{.{ .cache = cache, .rows = 1 }}, &.{token}, false, .all); + try self.run(&.{.{ .cache = cache, .rows = 1 }}, &.{token}, false, .all); + if (self.scratch.taps != null) cache.tap = .{ .start = cache.len - 1, .row = 0, .rows = 1, .gen = self.gen }; return try draw(self.gpa, cache.logits.slice(u16, c.vocab), stream.sampling, position, &.{}); } @@ -178,25 +287,43 @@ pub const Metal = struct { const self = cast(ptr); var segments: [st.batch_rows]fwd.Segment = undefined; var ids: [st.batch_rows]u32 = undefined; + var held: [st.batch_rows]fwd.Held = undefined; + var count: usize = 0; + var wait: ?fwd.Wait = null; var total: usize = 0; if (windows.len > segments.len or windows.len != out.len) return error.InvalidLfmWindows; for (windows, 0..) |w, i| { const rows = w.rows(); - if (w.parents != null or w.held != 0 or rows > st.window_rows or total + rows > st.batch_rows or w.positions.len != rows) return error.UnsupportedLfmWindow; + if (w.held != 0) { // the drafter's picks, copied into the window's rows on the GPU + const source = self.held_source orelse return error.UnsupportedLfmWindow; + if (total + rows > st.batch_rows) return error.UnsupportedLfmWindow; + if (source.get(source.ptr, w.stream)) |dev| { + held[count] = .{ .row = total + 1, .src = dev.src, .off = dev.off, .stride = dev.stride, .count = w.held }; + count += 1; + wait = if (wait) |x| (if (x.value >= dev.wait.value) x else dev.wait) else dev.wait; + @memset(ids[total + 1 ..][0..w.held], 0); + } else try source.host(source.ptr, w.stream, ids[total + 1 ..][0..w.held]); + } + if (w.parents != null or rows > st.window_rows or total + rows > st.batch_rows or w.positions.len != rows) return error.UnsupportedLfmWindow; if (w.masks.len != 0 and w.masks.len != rows * mask_words) return error.InvalidLfmMasks; const cache = try self.cacheOf(w.stream); for (w.positions, 0..) |p, r| if (p != cache.len + r + 1) return error.InvalidLfmPosition; segments[i] = .{ .cache = cache, .rows = rows }; ids[total] = w.pending; - @memcpy(ids[total + 1 ..][0..w.tokens.len], w.tokens); + @memcpy(ids[total + 1 + w.held ..][0..w.tokens.len], w.tokens); total += rows; } - try fwd.run(self.model, &self.scratch, segments[0..windows.len], ids[0..total], true, .all); + self.gen += 1; + try fwd.runHeld(self.model, &self.scratch, segments[0..windows.len], ids[0..total], true, .all, held[0..count], wait); + if (self.scratch.taps != null) for (segments[0..windows.len]) |seg| { + const last = seg.cache.last.?; + seg.cache.tap = .{ .start = last.start, .row = last.base, .rows = last.rows, .gen = self.gen }; + }; const logits = self.scratch.logits.slice(u16, total * c.vocab); const best = self.scratch.best.slice(u32, 2 * total); var base: usize = 0; for (windows, out) |w, o| { - if (o.sampled.len != w.rows() or o.drafts.len != w.tokens.len) return error.InvalidLfmOutputs; + if (o.sampled.len != w.rows() or o.drafts.len != w.rows() - 1) return error.InvalidLfmOutputs; for (o.sampled, w.positions, 0..) |*t, p, r| { const mask = if (w.masks.len > 0) w.masks[r * mask_words ..][0..mask_words] else &.{}; // A greedy row takes the GPU's pick, the host's draw on the same logits; a nonfinite row errors there. @@ -207,7 +334,7 @@ pub const Metal = struct { } t.* = try draw(self.gpa, logits[(base + r) * c.vocab ..][0..c.vocab], w.stream.sampling, p, mask); } - @memcpy(o.drafts, w.tokens); + @memcpy(o.drafts, self.scratch.ids.slice(u32, base + w.rows())[base + 1 ..][0 .. w.rows() - 1]); base += w.rows(); } } @@ -381,3 +508,15 @@ test "a masked draw is the full-vocabulary draw with the other tokens at -inf" { @memset(&mask, 0); try std.testing.expectError(error.NoAllowedLfmToken, draw(gpa, words, null, 0, &mask)); } + +test "measured shared rows price the batch without widening a stream's qualified window" { + var backend_: Metal = .{ .gpa = std.testing.allocator, .model = undefined, .options = .{ .capacity = 512 }, .scratch = undefined, .width = st.window_rows, .timed = 1, .shared_timed = 1 }; + backend_.costs[0] = .{ .width = 1, .ms = 2 }; + backend_.shared[0] = .{ .width = 17, .ms = 9 }; + var cfg = try lanes.Config.init(std.testing.allocator, backend_.facts(), st.window_rows, st.window_rows - 1); + defer cfg.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(u32, st.window_rows), cfg.family_width); + try std.testing.expectEqual(@as(?f64, 2), cfg.family_costs.get(1)); + try std.testing.expectEqual(@as(?f64, null), cfg.family_costs.get(17)); + try std.testing.expectEqual(@as(?f64, 9), cfg.shared_costs.get(17)); +} diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig new file mode 100644 index 0000000000..df55a38e9a --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -0,0 +1,578 @@ +//! LiquidAI's DSpark drafter on LFM2's taps: a stream's context keys, a block of rows, a Markov chain of drafts. +const std = @import("std"); +const mtl = @import("metal"); +const lanes = @import("lanes"); +const c = @import("config.zig"); +const dc = @import("dspark_config.zig"); +const dw = @import("dspark_weights.zig"); +const block = @import("dspark_block.zig"); +const st = @import("state.zig"); +const snapshot = @import("snapshot.zig"); +const Model = @import("model.zig").Model; +const backend = @import("backend.zig"); + +const dr = lanes.drafter; +const Stream = lanes.Stream; +const shared = mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked; + +/// Block rows one hold pass takes (the decode twin's rows), and context rows one absorb pass takes. +pub const pass_rows = 32; +pub const absorb_rows = 2048; +/// Verified rows a lane keeps on the host until its next hold absorbs them (a round's rows for no GPU work). +pub const stage_rows = 128; +/// The block counts timed at load, at every depth that fits one pass and the checkpoint's block. +const hold_grid = [_]u32{ 1, 2, 4, 8, 16 }; +const kw = c.kv_heads * c.head_dim; + +/// `bits`: 0 runs the bf16 matrices as stored, 8 quantizes them at load. +pub const Options = struct { capacity: usize, bits: u8 = 0 }; + +/// One stream's context keys and values a layer ([kv head][cap][head dim], as the target's attention reads them). +pub const Lane = struct { + mem: mtl.Buffer, // each layer's keys then values, then the staged rows (shared storage) + cap: usize, // positions each holds: the context and one block past it + stage_at: usize, // the staged rows' byte offset in `mem` + end: u64 = 0, // context rows absorbed: positions [0, end) + staged: u32 = 0, // tapped rows [end, end + staged), not absorbed yet + held: [16]u32 = @splat(0), + held_len: u32 = 0, + conf: [16]f32 = @splat(0), // each held draft's chance given the drafts before it landed (the confidence head) + conf_len: u32 = 0, // the drafts `conf` covers: the last settled hold's depth + slot: ?u32 = null, // its block's place in the last signalled hold, whose picks still hold its drafts + + /// Layer `layer`'s keys, or its values when `values`, in `mem`. + pub fn kv(l: *const Lane, layer: usize, values: bool) block.Ref { + return .{ .buf = l.mem, .off = (2 * layer + @intFromBool(values)) * l.cap * kw * 2 }; + } +}; + +/// The passes' working buffers, one pass at a time on the target's queue. +pub const Scratch = struct { + ids: mtl.Buffer, + h: mtl.Buffer, + x: mtl.Buffer, + r: mtl.Buffer, + y: mtl.Buffer, + q: mtl.Buffer, + k: mtl.Buffer, + v: mtl.Buffer, + queries: mtl.Buffer, + pm: mtl.Buffer, + pl: mtl.Buffer, + po: mtl.Buffer, + gate: mtl.Buffer, + up: mtl.Buffer, + act: mtl.Buffer, + logits: mtl.Buffer, + sums: mtl.Buffer, + fcx: mtl.Buffer, // an absorb pass's context rows: fc's output, normed in place + dump: mtl.Buffer, // the query heads lfm2_qk rotates from context rows, which no one reads + best: mtl.Buffer, // the Markov scan's best score and token of each threadgroup, by stream + at: mtl.Buffer, + picks: mtl.Buffer, + conf: mtl.Buffer, // the confidence logits, step by step, a pass's blocks each +}; + +pub const Drafter = struct { + gpa: std.mem.Allocator, + io: std.Io, + target: *Model, + config: dc.Config, + w: dw.Weights, + options: Options, + s: Scratch, + owned: std.ArrayList(mtl.Buffer) = .empty, + lanes: std.AutoHashMapUnmanaged(*Stream, *Lane) = .empty, + /// One stream's hold wall ms at full depth, and that over the block (the depth rule's cost a draft). + hold_ms: f64 = 0, + step_ms: f64 = 0, + holds: [2 * pass_rows]lanes.hold_price.HoldCost = undefined, // the power-of-two block counts sum to fewer than 2 * pass_rows depths + holds_len: usize = 0, + event: mtl.Event = undefined, + signalled: u64 = 0, + pending: ?mtl.CommandBuffer = null, + pass: [pass_rows]*Lane = undefined, // the signalled hold's lanes and depths, in its blocks' order + depths: [pass_rows]u32 = undefined, + pass_len: usize = 0, + + pub fn init(gpa: std.mem.Allocator, io: std.Io, target: *Model, dir: []const u8, options: Options) !*Drafter { + const d = try gpa.create(Drafter); + errdefer gpa.destroy(d); + d.* = .{ .gpa = gpa, .io = io, .target = target, .config = try dc.Config.read(gpa, io, dir), .w = undefined, .options = options, .s = undefined }; + d.event = try target.device.event(); + errdefer d.event.deinit(); + d.w = try dw.load(gpa, target.device, io, dir, &d.config, options.bits); + errdefer d.w.deinit(gpa); + try d.prepare(); + return d; + } + + fn prepare(d: *Drafter) !void { + errdefer { + for (d.owned.items) |b| b.deinit(); + d.owned.deinit(d.gpa); + } + try d.scratch(); + try d.warm(); + } + + pub fn deinit(d: *Drafter) void { + d.settle() catch {}; + d.event.deinit(); + var it = d.lanes.valueIterator(); + while (it.next()) |l| d.dropLane(l.*); + d.lanes.deinit(d.gpa); + for (d.owned.items) |b| b.deinit(); + d.owned.deinit(d.gpa); + d.w.deinit(d.gpa); + d.gpa.destroy(d); + } + + fn buffer(d: *Drafter, bytes: usize) !mtl.Buffer { + const b = try d.target.device.buffer(@max(bytes, 16), shared); + errdefer b.deinit(); + try d.owned.append(d.gpa, b); + return b; + } + + fn scratch(d: *Drafter) !void { + const r = pass_rows; + const wide = @max(pass_rows, absorb_rows); + const i = d.config.intermediate; + const partials = d.partRows() * st.subs * c.query_heads * st.chunks; + const sums = @max(absorb_rows * (d.config.tap_count * c.hidden / 64), r * (i / 64)); + d.s = .{ + .ids = try d.buffer(r * 4), + .h = try d.buffer(r * c.hidden * 2), + .x = try d.buffer(r * c.hidden * 2), + .r = try d.buffer(r * c.hidden * 2), + .y = try d.buffer(r * c.hidden * 2), + .q = try d.buffer(r * c.hidden * 2), + .k = try d.buffer(wide * kw * 2), + .v = try d.buffer(wide * kw * 2), + .queries = try d.buffer((r + st.attn_rows) * c.hidden * 2), + .pm = try d.buffer(partials * 4), + .pl = try d.buffer(partials * 4), + .po = try d.buffer(partials * c.head_dim * 4), + .gate = try d.buffer(r * i * 2), + .up = try d.buffer(r * i * 2), + .act = try d.buffer(r * i * 2), + .logits = try d.buffer(r * c.vocab * 2), + .sums = try d.buffer(sums * 4), + .fcx = try d.buffer(absorb_rows * c.hidden * 2), + .dump = try d.buffer(absorb_rows * c.hidden * 2), + .best = try d.buffer(r * block.scan_groups * 4), + .at = try d.buffer(r * block.scan_groups * 4), + .picks = try d.buffer(d.config.block * r * 8), + .conf = try d.buffer(d.config.block * r * 4), + }; + } + + /// Query rows the attention partials hold at once: a whole hold pass's, so its blocks attend side by side. + pub fn partRows(d: *const Drafter) usize { + return @max(pass_rows, d.config.block); + } + + /// The drafter's buffers that every round reads (weights and scratch), for the server's residency set. + pub fn buffers(d: *const Drafter, gpa: std.mem.Allocator) ![]mtl.Buffer { + return std.mem.concat(gpa, mtl.Buffer, &.{ d.w.buffers(), d.owned.items }); + } + + pub fn drafter(d: *Drafter) dr.Drafter { + return .{ .ptr = d, .vtable = &.{ .taps = tapsFn, .facts = factsFn, .open = openFn, .absorb = absorbFn, .hold = holdFn, .held = heldFn, .release = releaseFn, .chances = chancesFn } }; + } + + /// Whether holds compute the confidence head: the verify rows follow its chances, else the core's depth rule. + pub fn confident(d: *const Drafter) bool { + return d.w.confidence != null; + } + + fn cast(ptr: *anyopaque) *Drafter { + return @ptrCast(@alignCast(ptr)); + } + + fn tapsFn(ptr: *anyopaque) []const u32 { + const d = cast(ptr); + return d.config.taps[0..d.config.tap_count]; + } + + /// Acceptance a depth until a stream has its own: the card's 5.16 tokens a step at block 9 is 0.84 a depth. + const prior: [16]f64 = @splat(0.84); + + fn factsFn(ptr: *anyopaque) dr.Facts { + const d = cast(ptr); + var f: dr.Facts = .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .stream_bytes = d.laneBytes(d.options.capacity), .copy_first = true }; + f.chances = d.confident(); + f.hold_costs = d.holds[0..d.holds_len]; + return f; + } + + /// A held chain's chances: the confidence head's running product, or for a fixed window, ones that fall by depth. + fn chancesFn(ptr: *anyopaque, s: *Stream, out: []f64) anyerror!bool { + const d = cast(ptr); + const l = d.lanes.get(s) orelse return false; + if (!d.confident()) return false; + if (d.pending != null and l.slot != null and l.conf_len > 0) { + // the hold still runs: the lane's last settled confidences price it, so the host does not wait on the GPU + if (out.len > d.depths[l.slot.?]) return error.DraftHeldFewer; + var reach: f64 = 1; + for (out, 0..) |*o, j| { + reach *= l.conf[@min(j, l.conf_len - 1)]; + o.* = reach; + } + return true; + } + try d.settle(); + if (out.len > l.held_len) return error.DraftHeldFewer; + var reach: f64 = 1; + for (out, l.conf[0..out.len]) |*o, p| { + reach *= p; + o.* = reach; + } + return true; + } + + /// A stream's lane at `capacity` positions: its context keys and values a layer, and its staged rows. + pub fn laneBytes(d: *const Drafter, capacity: usize) usize { + return @intCast(d.keptBytes(capacity + d.config.block) + stage_rows * d.tapBytes()); + } + + /// A tapped row's bytes: every tap's hidden row, bf16. + pub fn tapBytes(d: *const Drafter) usize { + return d.config.tap_count * c.hidden * 2; + } + + /// The lane's staged rows as an absorb piece (null: none staged). + pub fn stagedPiece(l: *Lane) ?block.Piece { + if (l.staged == 0) return null; + return .{ .lane = l, .src = .{ .buf = l.mem, .off = l.stage_at }, .rows = l.staged, .p0 = l.end }; + } + + /// The lane's staged rows absorbed now, in a command buffer of their own. + fn flush(d: *Drafter, l: *Lane) !void { + const pc = stagedPiece(l) orelse return; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const run = block.begin(d); + try block.absorbPass(d, run.e, &.{pc}); + try block.end(run.cb, run.e); + l.end += l.staged; + l.staged = 0; + } + + fn openFn(ptr: *anyopaque, s: *Stream) anyerror!void { + const d = cast(ptr); + if (d.lanes.contains(s)) return error.DraftStreamOpen; + const want = s.prompt_len + s.max_new + 2 * d.config.block; + const l = try d.newLane(@min(d.options.capacity, (want + 63) / 64 * 64)); + errdefer d.dropLane(l); + try d.lanes.put(d.gpa, s, l); + } + + /// A lane in one buffer of laneBytes(capacity). + pub fn newLane(d: *Drafter, capacity: usize) !*Lane { + const l = try d.gpa.create(Lane); + errdefer d.gpa.destroy(l); + const at: usize = @intCast(d.keptBytes(capacity + d.config.block)); + l.* = .{ .mem = try d.target.device.buffer(d.laneBytes(capacity), shared), .cap = capacity + d.config.block, .stage_at = at }; + return l; + } + + pub fn dropLane(d: *Drafter, l: *Lane) void { + l.mem.deinit(); + d.gpa.destroy(l); + } + + fn releaseFn(ptr: *anyopaque, s: *Stream) void { + const d = cast(ptr); + d.unpass() catch {}; + if (d.lanes.fetchRemove(s)) |e| d.dropLane(e.value); + } + + fn heldFn(ptr: *anyopaque, streams: []const *Stream, out: []const []u32) anyerror!void { + const d = cast(ptr); + try d.settle(); + for (streams, out) |s, o| { + const l = d.lanes.get(s) orelse return error.DraftStreamUnknown; + if (o.len > l.held_len) return error.DraftHeldFewer; + @memcpy(o, l.held[0..o.len]); + } + } + + /// Bytes a lane's context at `at` takes beside a kept prompt state. + pub fn keptBytes(d: *const Drafter, at: usize) u64 { + return 2 * @as(u64, d.config.layers) * at * kw * 2; + } + + /// Stream `s`'s context copied into `snap` when its lane stands at the snapshot's position (a prompt chunk's end). + pub fn keepLane(d: *Drafter, s: *Stream, snap: *snapshot.Snap) !void { + const l = d.lanes.get(s) orelse return; + try d.settle(); + try d.flush(l); + if (l.end != snap.at) return; + const bytes = try d.gpa.alloc(u8, @intCast(d.keptBytes(snap.at))); + copyLane(d, l, snap.at, bytes, true); + snap.draft = bytes; + } + + /// The lane takes the context kept with the state its stream's prompt pass resumed at `start`. + fn resumeLane(d: *Drafter, s: *Stream, l: *Lane, start: u64) !void { + const saved = s.reuse.saved orelse return error.DraftContextGap; + const snap: *const snapshot.Snap = @ptrCast(@alignCast(saved)); + const bytes = snap.draft orelse return error.DraftContextGap; + if (s.cached != start or snap.at != start or l.end != 0 or start + d.config.block > l.cap) return error.DraftContextGap; + copyLane(d, l, snap.at, bytes, false); + l.end = start; + } + + /// Rows [0, at) of every layer's keys and values and `bytes` (`out`: the lane's into bytes), kv head by kv head. + fn copyLane(d: *const Drafter, l: *const Lane, at: usize, bytes: []u8, out: bool) void { + const n = at * c.head_dim * 2; + var off: usize = 0; + for (0..d.config.layers) |i| for ([_]block.Ref{ l.kv(i, false), l.kv(i, true) }) |r| for (0..c.kv_heads) |h| { + const live = (r.buf.contents() + r.off + h * l.cap * c.head_dim * 2)[0..n]; + if (out) @memcpy(bytes[off..][0..n], live) else @memcpy(live, bytes[off..][0..n]); + off += n; + }; + } + + /// Rows from `start` replace the context there: rows that fit the stage wait on the host for the next hold. + fn absorbFn(ptr: *anyopaque, items: []const dr.Absorb) anyerror!void { + const d = cast(ptr); + try d.settle(); + const width = d.tapBytes(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var run: ?@TypeOf(block.begin(d)) = null; + var pieces: [32]block.Piece = undefined; + var count: usize = 0; + var rows: usize = 0; + for (items) |it| { + const l = d.lanes.get(it.stream) orelse return error.DraftStreamUnknown; + const f = it.features; + if (f.space != .device or f.dtype != .bf16 or f.row_bytes != width or it.rows != null) return error.DraftFeaturesUnsupported; + if (it.start > l.end + l.staged) try d.resumeLane(it.stream, l, it.start); // a prompt pass that resumed + if (it.start + f.rows + d.config.block > l.cap) return error.DraftContextGap; + l.held_len = 0; + l.slot = null; + const src = mtl.Buffer{ .id = @ptrFromInt(f.buffer) }; + // rows from `start` replace what was staged past it; rows before the absorbed end go to the GPU now + const keep: u32 = if (it.start >= l.end) @intCast(it.start - l.end) else 0; + l.staged = @min(l.staged, keep); + if (it.start >= l.end and l.staged + f.rows <= stage_rows) { + const from = (src.contents() + f.offset)[0 .. f.rows * width]; + @memcpy((l.mem.contents() + l.stage_at + l.staged * width)[0 .. f.rows * width], from); + l.staged += f.rows; + continue; + } + if (run == null) run = block.begin(d); + const e = run.?.e; + if (stagedPiece(l)) |pc| { + if (count == pieces.len or rows + pc.rows > absorb_rows) { + try block.absorbPass(d, e, pieces[0..count]); + count = 0; + rows = 0; + } + pieces[count] = pc; + count += 1; + rows += pc.rows; + } + var at: usize = 0; + while (at < f.rows) { + if (rows == absorb_rows or count == pieces.len) { // the pass is full: encode it + try block.absorbPass(d, e, pieces[0..count]); + count = 0; + rows = 0; + } + const n = @min(absorb_rows - rows, f.rows - at); + pieces[count] = .{ .lane = l, .src = .{ .buf = src, .off = f.offset + at * width }, .rows = n, .p0 = it.start + at }; + count += 1; + rows += n; + at += n; + } + l.end = it.start + f.rows; + l.staged = 0; + } + const r = run orelse return; + // the other lanes' staged rows join the pass: one waited pass a stage's worth of rounds, not one a stream + var others = d.lanes.valueIterator(); + while (others.next()) |lp| { + const pc = stagedPiece(lp.*) orelse continue; + if (count == pieces.len or rows + pc.rows > absorb_rows) { + try block.absorbPass(d, r.e, pieces[0..count]); + count = 0; + rows = 0; + } + pieces[count] = pc; + count += 1; + rows += pc.rows; + lp.*.end += lp.*.staged; + lp.*.staged = 0; + } + if (count > 0) try block.absorbPass(d, r.e, pieces[0..count]); + try block.end(r.cb, r.e); + } + + /// Each stream's chain of `depth` drafts after `pending` at `position`, its block's rows in a pass of pass_rows. + fn holdFn(ptr: *anyopaque, items: []const dr.Hold) anyerror!void { + const d = cast(ptr); + try d.unpass(); + var at: usize = 0; + while (at < items.len) { + // as many streams as their blocks' rows fit a pass: each block is as long as the pass's deepest chain + var deepest: usize = 0; + var end = at; + while (end < items.len) : (end += 1) { + const rows = @max(deepest, items[end].depth); + if ((end - at + 1) * rows > pass_rows) break; + deepest = rows; + } + if (end == at) return error.DraftDepth; + var blocks: [pass_rows]block.Block = undefined; + const group = items[at..end]; + at = end; + for (group, blocks[0..group.len]) |it, *b| { + const l = d.lanes.get(it.stream) orelse return error.DraftStreamUnknown; + if (it.depth == 0 or it.depth > d.config.block or it.position == 0) return error.DraftDepth; + const anchor = it.position - 1; + if (l.end + l.staged != anchor or anchor + d.config.block > l.cap) return error.DraftContextGap; + b.* = .{ .lane = l, .pending = it.pending, .anchor = anchor, .depth = it.depth }; + } + if (group.len == items.len) { // one pass: signalled, not waited on + d.signalled += 1; + d.pending = try block.hold(d, blocks[0..group.len], .{ .event = d.event, .value = d.signalled }); + for (blocks[0..group.len], 0..) |b, bi| { + d.pass[bi] = b.lane; + d.depths[bi] = b.depth; + b.lane.slot = @intCast(bi); + b.lane.held_len = 0; + } + d.pass_len = group.len; + continue; + } + _ = try block.hold(d, blocks[0..group.len], null); + } + } + + /// The signalled hold waited on, its picks read into its lanes' held chains (nothing pending: nothing done). + fn settle(d: *Drafter) !void { + const cb = d.pending orelse return; + d.pending = null; + defer cb.release(); + cb.wait(); + if (cb.failure()) |msg| { + std.log.err("lfm2 dspark: a hold failed on the GPU: {s}", .{msg}); + return error.DraftGpuFailed; + } + d.priceHold((cb.gpuEnd() - cb.gpuStart()) * 1000); + try block.readPicks(d, d.pass[0..d.pass_len], d.depths[0..d.pass_len]); + } + + /// A settled hold's GPU ms against its price: the timed passes at its block counts move a tenth of the way to it. + pub fn priceHold(d: *Drafter, ms: f64) void { + const timed = d.holds[0..d.holds_len]; + if (timed.len == 0 or d.pass_len == 0 or !(ms > 0)) return; + const deepest = std.mem.max(u32, d.depths[0..d.pass_len]); + const priced = lanes.hold_price.holdMs(timed, d.pass_len, deepest); + if (!(priced > 0)) return; + const step = 1 + 0.1 * (ms / priced - 1); + var lo: u32 = 0; + var hi: u32 = std.math.maxInt(u32); + for (timed) |h| { + if (h.streams <= d.pass_len) lo = @max(lo, h.streams); + if (h.streams >= d.pass_len) hi = @min(hi, h.streams); + } + for (timed) |*h| if (h.streams >= lo and h.streams <= hi) { + h.ms *= step; + }; + } + + /// The signalled hold settled and its lanes' picks forgotten (a lane may be dropped, or a hold may overwrite). + fn unpass(d: *Drafter) !void { + defer { + for (d.pass[0..d.pass_len]) |l| l.slot = null; + d.pass_len = 0; + } + try d.settle(); + } + + /// Where stream `s`'s drafts from the last signalled hold lie in the picks, for a verify to copy on the GPU. + pub fn device(ptr: *anyopaque, s: *Stream) ?backend.Device { + const d = cast(ptr); + const l = d.lanes.get(s) orelse return null; + const slot = l.slot orelse return null; + return .{ .src = d.s.picks, .off = slot * 8, .stride = pass_rows * 2, .wait = .{ .event = d.event, .value = d.signalled } }; + } + + /// The target's verify reading the held drafts on the device, or as host tokens where it cannot. + pub fn heldSource(d: *Drafter) backend.HeldSource { + return .{ .ptr = d, .get = device, .host = hostHeld }; + } + + fn hostHeld(ptr: *anyopaque, s: *Stream, out: []u32) anyerror!void { + return heldFn(ptr, &.{s}, &.{out}); + } + + fn mibOf(bufs: []const mtl.Buffer) f64 { + var n: usize = 0; + for (bufs) |b| n += b.length(); + return @as(f64, @floatFromInt(n)) / (1024.0 * 1024.0); + } + + /// Holds on empty contexts: the pipelines compiled, then holds timed by blocks and depth (best of three). + fn warm(d: *Drafter) !void { + var made: [hold_grid[hold_grid.len - 1]]*Lane = undefined; + var n: usize = 0; + defer for (made[0..n]) |l| d.dropLane(l); + while (n < made.len) : (n += 1) made[n] = try d.newLane(64); + const l = made[0]; + var best: [2]f64 = @splat(std.math.floatMax(f64)); + for (0..4) |i| for ([_]u32{ 1, d.config.block }, &best) |depth, *b| { + const t0 = std.Io.Clock.awake.now(d.io); + _ = try block.hold(d, &.{.{ .lane = l, .pending = 1, .anchor = 0, .depth = depth }}, null); + const ms = @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(d.io)).toNanoseconds())) / 1e6; + if (i > 0) b.* = @min(b.*, ms); + }; + d.hold_ms = best[1]; + d.step_ms = best[1] / @as(f64, @floatFromInt(d.config.block)); + std.log.info("lfm2 dspark: a 1-draft hold takes {d:.2} ms, a {d}-draft hold {d:.2} ms", .{ best[0], d.config.block, best[1] }); + std.log.info("lfm2 dspark: weights {d:.1} MiB, scratch {d:.1} MiB, a stream {d} KiB a position and {d} KiB staged", .{ mibOf(d.w.buffers()), mibOf(d.owned.items), d.keptBytes(1) / 1024, stage_rows * d.tapBytes() / 1024 }); + var blocks: [hold_grid[hold_grid.len - 1]]block.Block = undefined; + for (made, &blocks) |lane, *b| b.* = .{ .lane = lane, .pending = 1, .anchor = 0, .depth = 1 }; + d.holds_len = 0; + for (hold_grid) |streams| for (1..@min(d.config.block, pass_rows / streams) + 1) |depth| { + for (blocks[0..streams]) |*b| b.depth = @intCast(depth); + var ms_best = std.math.floatMax(f64); + for (0..4) |i| { + const t0 = std.Io.Clock.awake.now(d.io); + _ = try block.hold(d, blocks[0..streams], null); + const ms = @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(d.io)).toNanoseconds())) / 1e6; + if (i > 0) ms_best = @min(ms_best, ms); + } + d.holds[d.holds_len] = .{ .streams = streams, .depth = @intCast(depth), .ms = ms_best }; + d.holds_len += 1; + }; + } +}; + +test "lfm2 dspark: scratch and warm allocation failures release buffers before their list" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + var target: Model = undefined; + target.device = device; + for ([_]usize{ 1, 3 }) |fail_index| { + var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = fail_index }); + var d: Drafter = .{ + .gpa = failing.allocator(), + .io = undefined, + .target = &target, + .config = try dc.Config.parse(std.testing.allocator, dc.stock), + .w = undefined, + .options = .{ .capacity = 64 }, + .s = undefined, + }; + try std.testing.expectError(error.OutOfMemory, d.prepare()); + try std.testing.expect(failing.has_induced_failure); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); + } +} diff --git a/zig/src/families/lfm2_moe/dspark_bf16_test.zig b/zig/src/families/lfm2_moe/dspark_bf16_test.zig new file mode 100644 index 0000000000..293549bde1 --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_bf16_test.zig @@ -0,0 +1,121 @@ +//! The DSpark drafter's bf16 kernels on the GPU against host references: the linears and the Markov scan over bf16 W2. +const std = @import("std"); +const mtl = @import("metal"); +const quant = @import("dspark_quant.zig"); +const base = @import("dspark_test.zig"); + +const Args = extern struct { rows: u32, n: u32, k: u32 }; + +/// y = x W^T through `key` on random bf16 data, each output within a bf16 step of the fp64 product. +fn linear(comptime key: []const u8, rows: usize, n: usize, k: usize, groups: [2]usize, threads: usize) !void { + var r = try base.Rig.init(); + defer r.deinit(); + const x = try r.buffer(u16, rows * k); + defer x.deinit(); + const w = try r.buffer(u16, n * k); + defer w.deinit(); + const y = try r.buffer(u16, rows * n); + defer y.deinit(); + base.noise(31, x.slice(u16, rows * k), 2); + base.noise(32, w.slice(u16, n * k), 0.5); + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get(key)); + e.setBuffer(x, 0, 0); + e.setBuffer(w, 0, 1); + e.setValue(Args{ .rows = @intCast(rows), .n = @intCast(n), .k = @intCast(k) }, 2); + e.setBuffer(y, 0, 3); + e.dispatchGroups(mtl.Size.of(groups[0], groups[1], 1), mtl.Size.of(threads, 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |_| return error.GpuFailed; + const xs = x.slice(u16, rows * k); + const ws = w.slice(u16, n * k); + const ys = y.slice(u16, rows * n); + for (0..rows) |i| for (0..n) |j| { + var dot: f64 = 0; + for (0..k) |t| dot += @as(f64, quant.toF32(xs[i * k + t])) * @as(f64, quant.toF32(ws[j * k + t])); + const got: f64 = quant.toF32(ys[i * n + j]); + try std.testing.expect(@abs(got - dot) <= @abs(dot) * 0x1p-7 + 1e-3); + }; +} + +test "lfm2 dspark: the bf16 rows kernel is x W^T for one row, a full 16 and a ragged 19 (two row groups)" { + for ([_]usize{ 1, 16, 19 }) |rows| try linear("lfm2_bf16_rows", rows, 100, 288, .{ 13, (rows + 15) / 16 }, 256); +} + +test "lfm2 dspark: the bf16 tile kernel is x W^T for a ragged 70 rows and 100 columns, k a multiple of 32" { + try linear("lfm2_bf16_mma", 70, 100, 288, .{ 2, 2 }, 128); +} + +test "lfm2 dspark: the bf16 tile kernel at the drafter's widths" { + try linear("lfm2_bf16_mma", 33, 512, 2048, .{ 8, 1 }, 128); +} + +test "lfm2 dspark: a Markov step over bf16 W2 picks the argmax of bf16(base + bf16(W2 . W1[prev])), a NaN row flagged" { + var r = try base.Rig.init(); + defer r.deinit(); + const rank = 256; + const vocab = 1300; + const streams = 4; + const groups = (vocab + 511) / 512; + const w1 = try r.buffer(u16, vocab * rank); + defer w1.deinit(); + base.noise(11, w1.slice(u16, vocab * rank), 1); + const w2 = try r.buffer(u16, vocab * rank); + defer w2.deinit(); + base.noise(12, w2.slice(u16, vocab * rank), 0.5); + const ids = try r.buffer(u32, 2 * streams); + defer ids.deinit(); + @memcpy(ids.slice(u32, 2 * streams), &[_]u32{ 17, 9, vocab - 1, 9, vocab + 5, 0, 3, 3 }); + const logits = try r.buffer(u16, streams * 2 * vocab); + defer logits.deinit(); + base.noise(14, logits.slice(u16, streams * 2 * vocab), 4); + logits.slice(u16, streams * 2 * vocab)[3 * 2 * vocab + 700] = 0x7fc0; + const best = try r.buffer(f32, streams * groups); + defer best.deinit(); + const at = try r.buffer(u32, streams * groups); + defer at.deinit(); + const picks = try r.buffer(u32, 2 * streams); + defer picks.deinit(); + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get("lfm2_markov_scan_bf16")); + for ([_]mtl.Buffer{ logits, w1, w2, ids }, 0..) |b, i| e.setBuffer(b, 0, i); + e.setValue([4]u32{ 2 * vocab, rank, vocab, 2 }, 4); + for ([_]mtl.Buffer{ best, at }, 5..) |b, i| e.setBuffer(b, 0, i); + e.dispatchGroups(mtl.Size.of(groups, streams, 1), mtl.Size.of(256, 1, 1)); + e.setPipeline(r.k.get("lfm2_markov_pick")); + e.setBuffer(best, 0, 0); + e.setBuffer(at, 0, 1); + e.setValue(@as(u32, groups), 2); + e.setBuffer(picks, 0, 3); + e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + const got = picks.slice(u32, 2 * streams); + const logit = logits.slice(u16, streams * 2 * vocab); + for (0..streams) |s| { + if (s == 3) { + try std.testing.expectEqual(@as(u32, 1), got[2 * s + 1]); + continue; + } + try std.testing.expectEqual(@as(u32, 0), got[2 * s + 1]); + const id = ids.slice(u32, 2 * streams)[2 * s]; + var top: f64 = -std.math.inf(f64); + var mine: f64 = 0; + for (0..vocab) |v| { + var dot: f64 = 0; + for (0..rank) |i| { + const e1: f64 = if (id < vocab) quant.toF32(w1.slice(u16, vocab * rank)[id * rank + i]) else 0; + dot += @as(f64, quant.toF32(w2.slice(u16, vocab * rank)[v * rank + i])) * e1; + } + const score: f64 = quant.toF32(quant.toBf16(quant.toF32(logit[s * 2 * vocab + v]) + quant.toF32(quant.toBf16(@floatCast(dot))))); + top = @max(top, score); + if (v == got[2 * s]) mine = score; + } + try std.testing.expect(got[2 * s] < vocab and mine >= top - @abs(top) * 0x1p-7); + } +} diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig new file mode 100644 index 0000000000..a737582c5e --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -0,0 +1,445 @@ +//! DSpark's passes on the target's queue: context rows into a lane's keys, and a block's rows into Markov drafts. +const std = @import("std"); +const mtl = @import("metal"); +const c = @import("config.zig"); +const st = @import("state.zig"); +const fwd = @import("forward.zig"); +const tile = @import("../../core/tile_qmm.zig"); +const am = @import("../../core/affine_mm.zig"); +const round = @import("tree_round_gpu"); +const ds = @import("dspark.zig"); +const dw = @import("dspark_weights.zig"); +const Drafter = ds.Drafter; +const Lane = ds.Lane; + +const kw = c.kv_heads * c.head_dim; +const QkArgs = extern struct { len: u32, cap: u32, eps: f32, log2_theta: f32 }; +const AttnArgs = extern struct { len: u32, cap: u32, scale: f32, subs: u32 }; +/// Partials a query row takes in the scratch (the kernels' QH x CHUNKS x SUBS). +const part_row = c.query_heads * st.chunks * st.subs; + +fn attnArgs(lane: *const Lane, keys: u64) AttnArgs { + return .{ .len = @intCast(keys - 1), .cap = @intCast(lane.cap), .scale = 1.0 / @sqrt(@as(f32, c.head_dim)), .subs = st.subs }; +} +const Pick = round.contract.Pick; +const Bf16Args = extern struct { rows: u32, n: u32, k: u32 }; +const MarkovArgs = extern struct { stride: u32, rank: u32, vocab: u32, prev_stride: u32 }; +const ConfidenceArgs = extern struct { n: u32, pass: u32, vocab: u32, rank: u32, hidden: u32 }; +/// Tokens a Markov scan threadgroup takes (the kernel's MARKOV_SPAN), and threadgroups a stream's row takes. +pub const scan_span = 512; +pub const scan_groups = (c.vocab + scan_span - 1) / scan_span; + +/// Rows a drafter projection takes on the scalar twin at most (faster there than the window or tensor units). +const scalar_rows = 8; +/// Rows from which the target's head (128,000 outputs) runs on the tensor units, where the GPU has them. +const head_rows = 5; + +/// Rows from which `w` runs on the tensor units: outputs of the hidden width or fewer stay on the twin to its rows. +fn naxFrom(w: tile.Matrix) usize { + if (w.n <= c.hidden) return tile.twin_rows + 1; + return if (w.n == c.vocab) head_rows else scalar_rows + 1; +} + +/// A buffer and byte offset. +pub const Ref = struct { buf: mtl.Buffer, off: usize = 0 }; + +/// One stream's block in a hold pass: rows [pending, mask x (block - 1)] at positions anchor .. anchor + block - 1. +pub const Block = struct { lane: *Lane, pending: u32, anchor: u64, depth: u32 }; + +const Enc = struct { + d: *Drafter, + e: mtl.ComputeEncoder, + bf16_rows_max: usize = 16, + conc: bool = false, // a concurrent encoder: each step waits for the writes before it at `sync` + + /// Later dispatches see the writes before this point (a no-op on a serial encoder). + fn sync(x: Enc) void { + if (x.conc) x.e.barrier(); + } + + fn pipe(x: Enc, comptime key: []const u8) void { + x.e.setPipeline(x.d.target.kernels.get(key)); + } + + /// y[rows] = x[rows] W^T: the M5's tensor units from naxFrom rows, else the twin (scalar to scalar_rows) or tiles. + fn project(x: Enc, w: dw.Linear, in: Ref, out: Ref, rows: usize) !void { + return x.projectAll(&.{w}, in, &.{out}, rows); + } + + /// y_i[rows] = x[rows] W_i^T on one input and shape: bf16 matrices straight, else one sums pass, products overlap. + fn projectAll(x: Enc, lins: []const dw.Linear, in: Ref, outs: []const Ref, rows: usize) !void { + if (lins[0] == .bf16) { + for (lins, outs) |lin, out| x.bf16(lin.bf16, in, out, rows); + return; + } + var buf: [4]tile.Matrix = undefined; + for (lins, 0..) |lin, i| buf[i] = lin.codes; + const ws = buf[0..lins.len]; + const t = &x.d.target.kernels.tiles; + const sums = Ref{ .buf = x.d.s.sums }; + const k = ws[0].k; + const nax = x.d.target.kernels.naxPipes(ws[0].bits); + if (rows >= naxFrom(ws[0])) if (nax) |p| { + am.rowSums(x.e, p, c.group, in, sums, @intCast(rows), @intCast(k)); + x.sync(); + for (ws, outs) |w, out| { + const q = .{ .w = Ref{ .buf = w.w, .off = w.w_off }, .s = Ref{ .buf = w.scales, .off = w.s_off }, .b = Ref{ .buf = w.biases, .off = w.b_off }, .n = @as(u32, @intCast(w.n)), .k = @as(u32, @intCast(w.k)) }; + am.dense(x.e, p, in, sums, q, out, @intCast(rows)); + } + return; + }; + if (rows <= tile.twin_rows and x.d.target.kernels.twin) { + // beside the tensor units the scalar twin takes every row count it has; elsewhere the window from 9 rows + const window: usize = if (nax != null) tile.twin_rows + 1 else scalar_rows + 1; + for (ws, outs) |w, out| try tile.encodeTwinFrom(x.e, t, w, in.buf, in.off, out.buf, out.off, rows, window); + return; + } + tile.encodeSums(x.e, t, k, in.buf, in.off, x.d.s.sums, rows); + x.sync(); + for (ws, outs) |w, out| try tile.encode(x.e, t, w, in.buf, in.off, x.d.s.sums, 0, out.buf, out.off, rows); + } + + /// y[rows] = x[rows] W^T over the released bf16 matrix: one 16-row pass of W, else 64x64 tiles. + fn bf16(x: Enc, m: dw.Bf16, in: Ref, out: Ref, rows: usize) void { + const args = Bf16Args{ .rows = @intCast(rows), .n = @intCast(m.n), .k = @intCast(m.k) }; + const few = rows <= x.bf16_rows_max; + if (few) x.pipe("lfm2_bf16_rows") else x.pipe("lfm2_bf16_mma"); + x.e.setBuffer(in.buf, in.off, 0); + x.e.setBuffer(m.w, 0, 1); + x.e.setBytes(std.mem.asBytes(&args), 2); + x.e.setBuffer(out.buf, out.off, 3); + if (few) { + x.e.dispatchGroups(mtl.Size.of((m.n + 7) / 8, (rows + 15) / 16, 1), mtl.Size.of(256, 1, 1)); + } else x.e.dispatchGroups(mtl.Size.of((m.n + 63) / 64, (rows + 63) / 64, 1), mtl.Size.of(128, 1, 1)); + } + + /// h += r when `add`, then out = norm(h) * gain: LFM2's own norm (the drafter's width and eps are the target's). + fn norm(x: Enc, h: mtl.Buffer, r: mtl.Buffer, gain: mtl.Buffer, add: bool, out: mtl.Buffer, rows: usize) void { + x.pipe("lfm2_norm"); + x.e.setBuffer(h, 0, 0); + x.e.setBuffer(r, 0, 1); + x.e.setBuffer(gain, 0, 2); + x.e.setValue(c.eps, 3); + x.e.setValue(@as(u32, @intFromBool(add)), 4); + x.e.setBuffer(out, 0, 5); + x.e.dispatchGroups(mtl.Size.of(rows, 1, 1), mtl.Size.of(c.hidden / 4, 1, 1)); + } + + /// Rows' queries normed and rotated into `queries`, keys and values into the lane's layer at positions from `len`. + fn qk(x: Enc, q: Ref, k: Ref, v: Ref, layer: usize, lane: *const Lane, len: u64, queries: Ref, rows: usize) void { + const ly = &x.d.w.layers[layer]; + x.pipe("lfm2_qk"); + x.e.setBuffer(q.buf, q.off, 0); + x.e.setBuffer(k.buf, k.off, 1); + x.e.setBuffer(v.buf, v.off, 2); + x.e.setBuffer(ly.q_norm, 0, 3); + x.e.setBuffer(ly.k_norm, 0, 4); + const args = QkArgs{ .len = @intCast(len), .cap = @intCast(lane.cap), .eps = c.eps, .log2_theta = std.math.log2(x.d.config.theta) }; + x.e.setBytes(std.mem.asBytes(&args), 5); + x.e.setBuffer(queries.buf, queries.off, 6); + x.e.setBuffer(lane.mem, lane.kv(layer, false).off, 7); + x.e.setBuffer(lane.mem, lane.kv(layer, true).off, 8); + x.e.dispatchGroups(mtl.Size.of(c.query_heads + c.kv_heads, rows, 1), mtl.Size.of(32, 1, 1)); + } + + /// The chunks' partials of query rows [row, row + rows) against the lane's keys, at partial row `part`. + fn partials(x: Enc, layer: usize, lane: *const Lane, row: usize, rows: usize, keys: u64, part: usize) void { + const s = &x.d.s; + const args = attnArgs(lane, keys); + const used = (keys + st.chunk_keys - 1) / st.chunk_keys * st.subs; + const at = part * part_row; + x.pipe("lfm2_attn_block"); + x.e.setBuffer(s.queries, row * c.hidden * 2, 0); + x.e.setBuffer(lane.mem, lane.kv(layer, false).off, 1); + x.e.setBuffer(lane.mem, lane.kv(layer, true).off, 2); + x.e.setBytes(std.mem.asBytes(&args), 3); + x.e.setBuffer(s.pm, at * 4, 4); + x.e.setBuffer(s.pl, at * 4, 5); + x.e.setBuffer(s.po, at * c.head_dim * 4, 6); + x.e.setValue(@as(u32, @intCast(rows)), 7); + x.e.dispatchGroups(mtl.Size.of((rows + 7) / 8, c.kv_heads, @intCast(used)), mtl.Size.of(128, 1, 1)); + } + + /// The partials from partial row `part` merged into the attention's output rows [row, row + rows). + fn merge(x: Enc, lane: *const Lane, row: usize, rows: usize, keys: u64, part: usize) void { + const s = &x.d.s; + const args = attnArgs(lane, keys); + const at = part * part_row; + x.pipe("lfm2_attn_block_merge"); + x.e.setBuffer(s.pm, at * 4, 0); + x.e.setBuffer(s.pl, at * 4, 1); + x.e.setBuffer(s.po, at * c.head_dim * 4, 2); + x.e.setBytes(std.mem.asBytes(&args), 3); + x.e.setBuffer(s.y, row * c.hidden * 2, 4); + x.e.dispatchGroups(mtl.Size.of(c.query_heads, rows, 1), mtl.Size.of(32, 1, 1)); + } +}; + +/// One stream's tapped rows in an absorb pass: `rows` rows at `src`, the context from position `p0`. +pub const Piece = struct { lane: *Lane, src: Ref, rows: usize, p0: u64 }; + +/// Each piece's rows into its lane's keys and values (fc, hidden_norm, k, v, k_norm, RoPE) in passes on `e`. +pub fn absorbPass(d: *Drafter, e: mtl.ComputeEncoder, pieces: []const Piece) !void { + return absorbOn(.{ .d = d, .e = e }, pieces); +} + +fn absorbOn(encoder: Enc, pieces: []const Piece) !void { + var x = encoder; + x.bf16_rows_max = 32; // preserve the context cache's original projection arithmetic + const d = x.d; + const s = &d.s; + var rows: usize = 0; + for (pieces) |pc| { + try x.project(d.w.fc, pc.src, .{ .buf = s.fcx, .off = rows * c.hidden * 2 }, pc.rows); + x.sync(); // the pieces' projections share the sums scratch + rows += pc.rows; + } + if (rows == 0 or rows > ds.absorb_rows) return error.DraftContextGap; + x.norm(s.fcx, s.fcx, d.w.hidden_norm, false, s.fcx, rows); // in place: a thread reads its values first + x.sync(); + for (0..d.config.layers) |i| { + const ly = &d.w.layers[i]; + try x.projectAll(&.{ ly.k, ly.v }, .{ .buf = s.fcx }, &.{ .{ .buf = s.k }, .{ .buf = s.v } }, rows); + x.sync(); + var at: usize = 0; + for (pieces) |pc| { + const q = Ref{ .buf = s.fcx, .off = at * c.hidden * 2 }; + x.qk(q, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, i, pc.lane, pc.p0, .{ .buf = s.dump }, pc.rows); + at += pc.rows; + } + x.sync(); + } +} + +/// A command buffer for absorb passes, and its wait. +pub fn begin(d: *Drafter) struct { cb: mtl.CommandBuffer, e: mtl.ComputeEncoder } { + const cb = d.target.queue.commandBuffer(); + return .{ .cb = cb, .e = cb.compute(.serial) }; +} + +pub fn end(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |msg| { + std.log.err("lfm2 dspark: command buffer failed: {s}", .{msg}); + return error.GpuFailed; + } +} + +/// Layer `layer`'s q, k, v and every block's attention (a lone block's last layer: `out` rows), on a concurrent `x`. +fn attendBlocks(x: Enc, layer: usize, blocks: []const Block, n: u32, rows: usize, out: usize) !void { + const s = &x.d.s; + const ly = &x.d.w.layers[layer]; + // the three projections, every block's keys, then waves of as many blocks as the partials hold at once + try x.projectAll(&.{ ly.q, ly.k, ly.v }, .{ .buf = s.x }, &.{ .{ .buf = s.q }, .{ .buf = s.k }, .{ .buf = s.v } }, rows); + x.sync(); + for (blocks, 0..) |b, bi| { + const at = bi * n; + x.qk(.{ .buf = s.q, .off = at * c.hidden * 2 }, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, layer, b.lane, b.anchor, .{ .buf = s.queries, .off = at * c.hidden * 2 }, n); + } + x.sync(); + const each = if (blocks.len == 1) out else n; + const wave = @max(1, x.d.partRows() / n); + var first: usize = 0; + while (first < blocks.len) : (first += wave) { + const last = @min(blocks.len, first + wave); + for (blocks[first..last], first..) |b, bi| x.partials(layer, b.lane, bi * n, each, b.anchor + n, (bi - first) * n); + x.sync(); + for (blocks[first..last], first..) |b, bi| x.merge(b.lane, bi * n, each, b.anchor + n, (bi - first) * n); + x.sync(); + } +} + +/// A GPU event a hold signals instead of being waited on, and the value it sets. +pub const Signal = struct { event: mtl.Event, value: u64 }; + +/// The blocks' drafts (backbone, then Markov chain) into the lanes' held chains; with `signal`, left for readPicks. +pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBuffer { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const s = &d.s; + var depth: u32 = 0; + for (blocks) |b| depth = @max(depth, b.depth); + // a block of the deepest chain's rows, at most the 9 the drafter was trained on + const n: u32 = depth; + const rows = blocks.len * n; + if (rows == 0 or rows > ds.pass_rows) return error.DraftTooManyStreams; + const ids = s.ids.slice(u32, ds.pass_rows); + for (0..blocks.len) |i| { + ids[i * n] = blocks[i].pending; + @memset(ids[i * n + 1 ..][0 .. n - 1], d.config.mask); + } + const steps = depth; + // past the last layer's attention a lone block carries only the rows its chain reads; several keep them all + const tail = if (blocks.len == 1) steps else rows; + const cb = d.target.queue.commandBuffer(); + // one concurrent encoder: independent dispatches overlap, each dependent step waits at a barrier + const x = Enc{ .d = d, .e = cb.compute(.concurrent), .conc = true }; + // the blocks' staged context rows first, in the same command buffer + var staged: [ds.pass_rows]Piece = undefined; + var count: usize = 0; + var held: usize = 0; + for (blocks) |b| if (ds.Drafter.stagedPiece(b.lane)) |pc| { + if (held + pc.rows > ds.absorb_rows) { + try absorbOn(x, staged[0..count]); + count = 0; + held = 0; + } + staged[count] = pc; + count += 1; + held += pc.rows; + }; + if (count > 0) try absorbOn(x, staged[0..count]); + for (blocks) |b| { + b.lane.end += b.lane.staged; + b.lane.staged = 0; + } + const t = d.target; + const emb = t.weights.embedding; + x.e.setPipeline(t.kernels.embed(emb.bits)); + x.e.setBuffer(s.ids, 0, 0); + x.e.setBuffer(emb.w, emb.w_off, 1); + x.e.setBuffer(emb.scales, emb.s_off, 2); + x.e.setBuffer(emb.biases, emb.b_off, 3); + x.e.setBuffer(s.h, 0, 4); + const units = c.hidden / @as(usize, if (emb.bits == 6) 4 else 8 / emb.bits); + x.e.dispatchThreads(mtl.Size.of(units, rows, 1), mtl.Size.of(256, 1, 1)); + x.sync(); + for (0..d.config.layers) |i| { + const ly = &d.w.layers[i]; + const out = if (i + 1 == d.config.layers) tail else rows; + x.norm(s.h, s.r, ly.input_norm, i > 0, s.x, rows); + x.sync(); + try attendBlocks(x, i, blocks, n, rows, out); + try x.project(ly.o, .{ .buf = s.y }, .{ .buf = s.r }, out); + x.sync(); + x.norm(s.h, s.r, ly.post_norm, true, s.x, out); + x.sync(); + try x.projectAll(&.{ ly.gate, ly.up }, .{ .buf = s.x }, &.{ .{ .buf = s.gate }, .{ .buf = s.up } }, out); + x.sync(); + x.pipe("lfm2_swiglu"); + x.e.setBuffer(s.gate, 0, 0); + x.e.setBuffer(s.up, 0, 1); + x.e.setBuffer(s.act, 0, 2); + x.e.setValue(@as(u32, @intCast(out * d.config.intermediate)), 3); + x.e.dispatchThreads(mtl.Size.of(out * d.config.intermediate / 8, 1, 1), mtl.Size.of(256, 1, 1)); + x.sync(); + try x.project(ly.down, .{ .buf = s.act }, .{ .buf = s.r }, out); + x.sync(); + } + x.norm(s.h, s.r, d.w.norm, true, s.x, tail); + x.sync(); + try x.project(.{ .codes = fwd.matrix(emb) }, .{ .buf = s.x }, .{ .buf = s.logits }, tail); + x.sync(); + try markov(x, blocks.len, n, steps); + if (d.confident()) confidence(x, blocks.len, n, steps); + const e = x.e; + if (signal) |sg| { // committed unwaited, the command buffer returned retained + e.end(); + cb.signalEvent(sg.event, sg.value); + const kept = cb.retain(); + cb.commit(); + return kept; + } + try end(cb, e); + const picks = s.picks.slice(Pick, d.config.block * ds.pass_rows); + for (blocks, 0..) |b, bi| { + for (b.lane.held[0..steps], 0..) |*h, i| { + const p = picks[i * ds.pass_rows + bi]; + if (p.nonfinite != 0 or p.token >= c.vocab) return error.DraftNonfinite; + h.* = p.token; + } + b.lane.held_len = b.depth; + } + for (blocks, 0..) |b, bi| readConfidence(d, b.lane, b.depth, bi); + return null; +} + +/// Lane `l`'s drafts' chances from slot `bi` of the last hold's logits: sigmoid(logit + bias); none: left as they are. +fn readConfidence(d: *Drafter, l: *Lane, depth: u32, bi: usize) void { + if (!d.confident()) return; + const logits = d.s.conf.slice(f32, d.config.block * ds.pass_rows); + for (l.conf[0..depth], 0..) |*p, i| p.* = 1 / (1 + @exp(-(logits[i * ds.pass_rows + bi] + d.w.confidence_bias))); + l.conf_len = depth; +} + +/// A finished hold's picks into the lanes' held chains; a nonfinite or invalid pick is refused, as when synchronous. +pub fn readPicks(d: *Drafter, lanes: []const *Lane, depths: []const u32) !void { + const picks = d.s.picks.slice(Pick, d.config.block * ds.pass_rows); + var chains: [ds.pass_rows][]u32 = undefined; + for (lanes, chains[0..lanes.len]) |l, *ch| ch.* = &l.held; + try copyPicks(picks, chains[0..lanes.len], depths); + for (lanes, depths, 0..) |l, depth, bi| { + l.held_len = depth; + readConfidence(d, l, depth, bi); + } +} + +/// Step i of block bi is picks[i * ds.pass_rows + bi]: each block's first `depth` picks into its chain. +pub fn copyPicks(picks: []const Pick, chains: []const []u32, depths: []const u32) !void { + for (chains, depths, 0..) |chain, depth, bi| { + for (chain[0..depth], 0..) |*h, i| { + const p = picks[i * ds.pass_rows + bi]; + if (p.nonfinite != 0 or p.token >= c.vocab) return error.DraftNonfinite; + h.* = p.token; + } + } +} + +/// Each stream's draft i confidence logit (the final norm's row i, W1 of the token before draft i), for readConfidence. +fn confidence(x: Enc, streams: usize, n: u32, depth: u32) void { + const d = x.d; + const e = x.e; + x.pipe("lfm2_dspark_confidence"); + e.setBuffer(d.s.x, 0, 0); + e.setBuffer(d.w.w1, 0, 1); + e.setBuffer(d.s.ids, 0, 2); + e.setBuffer(d.s.picks, 0, 3); + e.setBuffer(d.w.confidence.?, 0, 4); + const args = ConfidenceArgs{ .n = n, .pass = ds.pass_rows, .vocab = c.vocab, .rank = d.config.rank, .hidden = c.hidden }; + e.setBytes(std.mem.asBytes(&args), 5); + e.setBuffer(d.s.conf, 0, 6); + e.dispatchGroups(mtl.Size.of(depth, streams, 1), mtl.Size.of(256, 1, 1)); +} + +/// Step i of each stream: its row i's logits plus W2 W1[previous draft] (the pending token first), then the argmax. +fn markov(x: Enc, streams: usize, n: u32, depth: u32) !void { + const d = x.d; + const s = &d.s; + const e = x.e; + for (0..depth) |i| { + // step i's previous token: the pending one (each block's first id), then step i - 1's pick + switch (d.w.w2) { + .codes => x.pipe("lfm2_markov_scan"), + .bf16 => x.pipe("lfm2_markov_scan_bf16"), + } + e.setBuffer(s.logits, i * c.vocab * 2, 0); + e.setBuffer(d.w.w1, 0, 1); + e.setBuffer(switch (d.w.w2) { + .codes => |m| m.w, + .bf16 => |m| m.w, + }, switch (d.w.w2) { + .codes => |m| m.w_off, + .bf16 => 0, + }, 2); + if (i == 0) e.setBuffer(s.ids, 0, 3) else e.setBuffer(s.picks, (i - 1) * ds.pass_rows * @sizeOf(Pick), 3); + const args = MarkovArgs{ .stride = n * c.vocab, .rank = d.config.rank, .vocab = c.vocab, .prev_stride = if (i == 0) n else @sizeOf(Pick) / 4 }; + e.setBytes(std.mem.asBytes(&args), 4); + e.setBuffer(s.best, 0, 5); + e.setBuffer(s.at, 0, 6); + if (d.w.w2 == .codes) { + e.setBuffer(d.w.w2.codes.scales, d.w.w2.codes.s_off, 7); + e.setBuffer(d.w.w2.codes.biases, d.w.w2.codes.b_off, 8); + } + e.dispatchGroups(mtl.Size.of(scan_groups, streams, 1), mtl.Size.of(256, 1, 1)); + x.sync(); + x.pipe("lfm2_markov_pick"); + e.setBuffer(s.best, 0, 0); + e.setBuffer(s.at, 0, 1); + e.setValue(@as(u32, scan_groups), 2); + e.setBuffer(s.picks, i * ds.pass_rows * @sizeOf(Pick), 3); + e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1)); + x.sync(); + } +} diff --git a/zig/src/families/lfm2_moe/dspark_config.zig b/zig/src/families/lfm2_moe/dspark_config.zig new file mode 100644 index 0000000000..68acc6d30d --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_config.zig @@ -0,0 +1,142 @@ +//! LiquidAI's DSpark drafter for LFM2.5-24B-A2B: its config admitted, any other shape refused with its reason. +const std = @import("std"); +const c = @import("config.zig"); + +/// The geometry the drafter's kernels take: the target's attention shape, five taps at most eight layers. +pub const Config = struct { + taps: [8]u32 = @splat(0), + tap_count: u32, + layers: u32, + intermediate: u32, + /// Rows a block takes: the pending token and block - 1 mask tokens, one draft a row. + block: u32, + mask: u32, + rank: u32, + theta: f32, + + pub fn read(gpa: std.mem.Allocator, io: std.Io, dir: []const u8) !Config { + const path = try std.fs.path.join(gpa, &.{ dir, "config.json" }); + defer gpa.free(path); + const bytes = try std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(1 << 20)); + defer gpa.free(bytes); + return parse(gpa, bytes); + } + + pub fn parse(gpa: std.mem.Allocator, bytes: []const u8) !Config { + var parsed = try std.json.parseFromSlice(std.json.Value, gpa, bytes, .{}); + defer parsed.deinit(); + const o = if (parsed.value == .object) parsed.value.object else return error.BadDraftConfig; + const archs = o.get("architectures") orelse return error.BadDraftConfig; + if (archs != .array or archs.array.items.len != 1 or archs.array.items[0] != .string) return error.BadDraftConfig; + if (!std.mem.eql(u8, archs.array.items[0].string, "Lfm2DSparkDraftModel")) return error.NotLfmDSpark; + const d = if (o.get("dflash_config")) |v| (if (v == .object) v.object else return error.BadDraftConfig) else return error.BadDraftConfig; + var out: Config = .{ + .tap_count = 0, + .layers = try small(o, "num_hidden_layers"), + .intermediate = try small(o, "intermediate_size"), + .block = try small(o, "block_size"), + .mask = try small(d, "mask_token_id"), + .rank = try small(o, "markov_rank"), + .theta = @floatCast(try number(o, "rope_theta")), + }; + // the target's widths: the drafter shares its embedding, head and attention kernels + if (try small(o, "hidden_size") != c.hidden or try small(o, "vocab_size") != c.vocab) return error.DraftForAnotherTarget; + if (try small(d, "num_target_layers") != c.layers) return error.DraftForAnotherTarget; + const heads = try small(o, "num_attention_heads"); + const kv = try small(o, "num_key_value_heads"); + if (heads != c.query_heads or kv != c.kv_heads or try small(o, "head_dim") != c.head_dim) return error.UnsupportedDraftShape; + if (@as(f32, @floatCast(try number(o, "rms_norm_eps"))) != c.eps) return error.UnsupportedDraftShape; + const taps = d.get("target_layer_ids") orelse return error.BadDraftConfig; + if (taps != .array or taps.array.items.len == 0 or taps.array.items.len > out.taps.len) return error.BadDraftConfig; + for (taps.array.items, 0..) |t, i| { + if (t != .integer or t.integer < 0) return error.BadDraftConfig; + if (t.integer >= c.layers) return error.DraftForAnotherTarget; + out.taps[i] = @intCast(t.integer); + } + out.tap_count = @intCast(taps.array.items.len); + try kinds(o, out.layers); + // the block's MLP is the fixed SiLU gate + const act = o.get("hidden_act") orelse return error.BadDraftConfig; + if (act != .string or !std.mem.eql(u8, act.string, "silu")) return error.UnsupportedDraftActivation; + // the vendored defaults decide the block: non-causal attention, interleaved RoPE, a plain Markov head + if (o.get("is_causal") != null) return error.UnsupportedDraftMask; + const neox = o.get("rope_is_neox_style") orelse return error.BadDraftConfig; + if (neox != .bool or neox.bool) return error.UnsupportedDraftRope; + const head = o.get("markov_head_type") orelse return error.BadDraftConfig; + if (head != .string or !std.mem.eql(u8, head.string, "vanilla")) return error.UnsupportedMarkovHead; + if (o.get("sample_from_anchor")) |v| if (v != .bool or !v.bool) return error.UnsupportedDraftAnchor; + if (out.layers == 0 or out.layers > 8 or out.block < 2 or out.block > 16) return error.UnsupportedDraftShape; + if (out.intermediate == 0 or out.intermediate % 64 != 0 or out.rank == 0 or out.rank > 256 or out.rank % 64 != 0) return error.UnsupportedDraftShape; + if (out.mask >= c.vocab or !std.math.isFinite(out.theta) or !(out.theta > 0)) return error.BadDraftConfig; + return out; + } +}; + +/// Every layer full attention (the block's mask has no sliding window). +fn kinds(o: std.json.ObjectMap, layers: u32) !void { + const v = o.get("layer_types") orelse return; + if (v != .array or v.array.items.len != layers) return error.BadDraftConfig; + for (v.array.items) |k| if (k != .string or !std.mem.eql(u8, k.string, "full_attention")) return error.UnsupportedDraftMask; +} + +fn small(o: std.json.ObjectMap, key: []const u8) !u32 { + const v = o.get(key) orelse return error.BadDraftConfig; + if (v != .integer or v.integer < 0 or v.integer > std.math.maxInt(u32)) return error.BadDraftConfig; + return @intCast(v.integer); +} + +fn number(o: std.json.ObjectMap, key: []const u8) !f64 { + const v = o.get(key) orelse return error.BadDraftConfig; + return switch (v) { + .integer => |i| @floatFromInt(i), + .float => |f| f, + else => error.BadDraftConfig, + }; +} + +/// LiquidAI/LFM2.5-24B-A2B-DSpark's config.json at e052e5b1dd10, as released. +pub const stock = + \\{"architectures":["Lfm2DSparkDraftModel"],"model_type":"qwen3","dtype":"bfloat16","hidden_size":2048, + \\"num_hidden_layers":5,"num_attention_heads":32,"num_key_value_heads":8,"head_dim":64,"intermediate_size":6144, + \\"hidden_act":"silu","rms_norm_eps":1e-05,"vocab_size":128000,"rope_theta":10000000.0, + \\"max_position_embeddings":131072,"layer_types":["full_attention","full_attention","full_attention", + \\"full_attention","full_attention"],"block_size":9,"dflash_config":{"mask_token_id":125017, + \\"target_layer_ids":[2,10,18,26,34],"num_target_layers":40},"markov_rank":256,"rope_is_neox_style":false, + \\"enable_confidence_head":true,"markov_head_type":"vanilla"} +; + +fn edited(from: []const u8, to: []const u8) !Config { + const a = std.testing.allocator; + const text = try std.mem.replaceOwned(u8, a, stock, from, to); + defer a.free(text); + return Config.parse(a, text); +} + +test "lfm2 dspark: the released config is admitted with its taps, block, mask token and Markov rank" { + const cfg = try Config.parse(std.testing.allocator, stock); + try std.testing.expectEqualSlices(u32, &.{ 2, 10, 18, 26, 34 }, cfg.taps[0..cfg.tap_count]); + try std.testing.expectEqual(@as(u32, 5), cfg.layers); + try std.testing.expectEqual(@as(u32, 9), cfg.block); + try std.testing.expectEqual(@as(u32, 125017), cfg.mask); + try std.testing.expectEqual(@as(u32, 256), cfg.rank); + try std.testing.expectEqual(@as(u32, 6144), cfg.intermediate); + try std.testing.expectEqual(@as(f32, 1e7), cfg.theta); +} + +test "lfm2 dspark: other drafters and shapes are refused with their reason" { + try std.testing.expectError(error.NotLfmDSpark, edited("Lfm2DSparkDraftModel", "DFlashDraftModel")); + try std.testing.expectError(error.DraftForAnotherTarget, edited("\"hidden_size\":2048", "\"hidden_size\":4096")); + try std.testing.expectError(error.DraftForAnotherTarget, edited("\"num_target_layers\":40", "\"num_target_layers\":24")); + try std.testing.expectError(error.DraftForAnotherTarget, edited("26,34]", "26,44]")); + try std.testing.expectError(error.UnsupportedDraftShape, edited("\"head_dim\":64", "\"head_dim\":128")); + try std.testing.expectError(error.UnsupportedDraftActivation, edited("\"silu\"", "\"gelu\"")); + try std.testing.expectError(error.UnsupportedDraftRope, edited("\"rope_is_neox_style\":false", "\"rope_is_neox_style\":true")); + try std.testing.expectError(error.UnsupportedMarkovHead, edited("\"vanilla\"", "\"gated\"")); + try std.testing.expectError(error.UnsupportedDraftMask, edited("\"full_attention\"]", "\"sliding_attention\"]")); + try std.testing.expectError(error.UnsupportedDraftMask, edited("\"block_size\"", "\"is_causal\":true,\"block_size\"")); + try std.testing.expectError(error.UnsupportedDraftAnchor, edited("\"block_size\"", "\"sample_from_anchor\":false,\"block_size\"")); + try std.testing.expectError(error.UnsupportedDraftShape, edited("\"block_size\":9", "\"block_size\":17")); + try std.testing.expectError(error.UnsupportedDraftShape, edited("\"markov_rank\":256", "\"markov_rank\":512")); + try std.testing.expectError(error.BadDraftConfig, edited("10000000.0", "1e100")); + try std.testing.expectError(error.BadDraftConfig, edited("\"mask_token_id\":125017", "\"mask_token_id\":128000")); +} diff --git a/zig/src/families/lfm2_moe/dspark_quant.zig b/zig/src/families/lfm2_moe/dspark_quant.zig new file mode 100644 index 0000000000..a34b52dd20 --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_quant.zig @@ -0,0 +1,118 @@ +//! BF16 drafter rows quantized at load to MLX's affine codes in groups of 64, the layout core/tile_qmm reads. +const std = @import("std"); + +/// MLX's affine codes of 64 bf16 values at `bits`: bias the larger edge, scale |edge| / round(|edge| / step). +pub fn group(x: []const u16, bits: u8, codes: []u8, scale: *u16, bias: *u16) void { + var lo: f32 = std.math.inf(f32); + var hi: f32 = -std.math.inf(f32); + for (x) |v| { + lo = @min(lo, toF32(v)); + hi = @max(hi, toF32(v)); + } + const bins: f32 = @floatFromInt((@as(u32, 1) << @intCast(bits)) - 1); + const step = @max(hi - lo, 1e-7) / bins; + const from_min = @abs(lo) > @abs(hi); + const edge = if (from_min) lo else hi; + const q0 = @round(@abs(edge) / step); + const s_f: f32 = if (q0 == 0) step else (if (from_min) @abs(edge) else -@abs(edge)) / q0; + scale.* = toBf16(s_f); + bias.* = toBf16(if (q0 == 0) lo else edge); + const s = toF32(scale.*); + const b = toF32(bias.*); + @memset(codes, 0); + for (x, 0..) |v, i| { + const q: u8 = @intFromFloat(std.math.clamp(@round((toF32(v) - b) / s), 0, bins)); + if (bits == 8) codes[i] = q else codes[i / 2] |= q << @intCast((i % 2) * 4); + } +} + +pub fn toF32(v: u16) f32 { + return @bitCast(@as(u32, v) << 16); +} + +/// Round to nearest even, as MLX casts. +pub fn toBf16(f: f32) u16 { + const b: u32 = @bitCast(f); + if ((b & 0x7f80_0000) == 0x7f80_0000) return @truncate(b >> 16); + return @truncate((b + 0x7fff + ((b >> 16) & 1)) >> 16); +} + +/// A matrix's codes [n][k * bits / 8], scales and biases [n][k / 64], each written by one thread's rows. +pub const Job = struct { + src: []const u16, + k: usize, + bits: u8, + codes: []u8, + scales: []u16, + biases: []u16, + fn run(j: Job, from: usize, to: usize) void { + const groups = j.k / 64; + const per = 64 * @as(usize, j.bits) / 8; + for (from..to) |r| for (0..groups) |g| { + const at = r * groups + g; + group(j.src[r * j.k + g * 64 ..][0..64], j.bits, j.codes[at * per ..][0..per], &j.scales[at], &j.biases[at]); + }; + } + + /// The job over its rows on every CPU (at most 16 threads). + pub fn all(j: Job, gpa: std.mem.Allocator, rows: usize) !void { + const threads = @min(std.Thread.getCpuCount() catch 1, 16); + const per = (rows + threads - 1) / threads; + const handles = try gpa.alloc(std.Thread, threads); + defer gpa.free(handles); + var spawned: usize = 0; + defer for (handles[0..spawned]) |h| h.join(); + var at: usize = 0; + while (at < rows) : (at += per) { + handles[spawned] = try std.Thread.spawn(.{}, run, .{ j, at, @min(rows, at + per) }); + spawned += 1; + } + } +}; + +/// Head `h`'s interleaved RoPE pairs (2i, 2i + 1) as half-split pairs (i, i + dim / 2): row h * dim + j's source. +pub fn halfSplit(row: usize, dim: usize) usize { + const h = row / dim; + const j = row % dim; + const half = dim / 2; + return h * dim + if (j < half) 2 * j else 2 * (j - half) + 1; +} + +test "lfm2 dspark: 4- and 8-bit codes keep the larger edge exact, as MLX quantizes" { + var x: [64]u16 = undefined; + for (&x, 0..) |*v, i| v.* = toBf16(@as(f32, @floatFromInt(i)) * 0.125 - 2.0); + for ([_]u8{ 4, 8 }) |bits| { + var codes: [64]u8 = undefined; + var s: u16 = undefined; + var b: u16 = undefined; + group(&x, bits, codes[0 .. 64 * @as(usize, bits) / 8], &s, &b); + try std.testing.expect(toF32(s) < 0 and toF32(b) == 5.875); // the max is the edge: codes descend from it + const last: u8 = if (bits == 8) codes[63] else codes[31] >> 4; + try std.testing.expectEqual(@as(u8, 0), last); + for (x, 0..) |v, i| { + const q: u8 = if (bits == 8) codes[i] else (codes[i / 2] >> @intCast((i % 2) * 4)) & 0xf; + const back = toF32(s) * @as(f32, @floatFromInt(q)) + toF32(b); + try std.testing.expect(@abs(back - toF32(v)) <= @abs(toF32(s)) * 0.51); + } + } +} + +test "lfm2 dspark: half-split rows rotated by the half-split rule equal interleaved pairs rotated, row for row" { + const dim = 64; + var x: [2 * dim]f32 = undefined; + for (&x, 0..) |*v, i| v.* = @sin(@as(f32, @floatFromInt(i)) * 0.37) * 3; + var moved: [2 * dim]f32 = undefined; + for (&moved, 0..) |*v, r| v.* = x[halfSplit(r, dim)]; + const pos: f32 = 1234; + for (0..2) |h| for (0..dim / 2) |i| { + const angle = pos * std.math.pow(f32, 1e7, -@as(f32, @floatFromInt(i)) / (dim / 2)); + const cs = @cos(angle); + const sn = @sin(angle); + const a = x[h * dim + 2 * i]; + const b = x[h * dim + 2 * i + 1]; + const l = moved[h * dim + i]; + const r = moved[h * dim + i + dim / 2]; + try std.testing.expectEqual(a * cs - b * sn, l * cs - r * sn); + try std.testing.expectEqual(a * sn + b * cs, l * sn + r * cs); + }; +} diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig new file mode 100644 index 0000000000..043a590353 --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_test.zig @@ -0,0 +1,272 @@ +//! The DSpark drafter's kernels on the GPU against host references: taps, the Markov step, half-split RoPE. +const std = @import("std"); +const mtl = @import("metal"); +const c = @import("config.zig"); +const kernels = @import("kernels.zig"); +const quant = @import("dspark_quant.zig"); +const block = @import("dspark_block.zig"); +const Pick = @import("tree_round_gpu").contract.Pick; +const ds = @import("dspark.zig"); +const lanes = @import("lanes"); + +const shared = mtl.ResourceOptions.shared; + +pub const Rig = struct { + device: mtl.Device, + queue: mtl.Queue, + k: kernels.Kernels, + + pub fn init() !Rig { + const device = mtl.Device.init() catch return error.SkipZigTest; + errdefer device.deinit(); + const queue = try device.queue(); + errdefer queue.deinit(); + return .{ .device = device, .queue = queue, .k = try kernels.load(device) }; + } + + pub fn deinit(r: *Rig) void { + r.k.deinit(); + r.queue.deinit(); + r.device.deinit(); + } + + pub fn buffer(r: *Rig, comptime T: type, n: usize) !mtl.Buffer { + const b = try r.device.buffer(n * @sizeOf(T), shared); + @memset(b.slice(u8, n * @sizeOf(T)), 0); + return b; + } + + fn run(r: *Rig, comptime key: []const u8, bufs: []const mtl.Buffer, value: anytype, at: usize, grid: [2]usize) !void { + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get(key)); + for (bufs, 0..) |b, i| e.setBuffer(b, 0, if (i >= at) i + 1 else i); + e.setValue(value, at); + e.dispatchThreads(mtl.Size.of(grid[0], grid[1], 1), mtl.Size.of(@min(grid[0], 256), 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |_| return error.GpuFailed; + } +}; + +pub fn noise(seed: u64, out: []u16, scale: f32) void { + var prng = std.Random.DefaultPrng.init(seed); + for (out) |*v| v.* = quant.toBf16((prng.random().float(f32) * 2 - 1) * scale); +} + +test "lfm2 dspark: the tap kernel copies each row's hidden values into its tap's columns, the rest untouched" { + var r = try Rig.init(); + defer r.deinit(); + const rows = 5; + const taps = 3; + const h = try r.buffer(u16, rows * c.hidden); + defer h.deinit(); + const out = try r.buffer(u16, rows * taps * c.hidden); + defer out.deinit(); + noise(7, h.slice(u16, rows * c.hidden), 4); + try r.run("lfm2_tap", &.{ h, out }, [2]u32{ taps * c.hidden, c.hidden }, 2, .{ c.hidden / 8, rows }); + const want = h.slice(u16, rows * c.hidden); + const got = out.slice(u16, rows * taps * c.hidden); + for (0..rows) |row| { + try std.testing.expectEqualSlices(u16, want[row * c.hidden ..][0..c.hidden], got[(row * taps + 1) * c.hidden ..][0..c.hidden]); + for (got[row * taps * c.hidden ..][0..c.hidden]) |v| try std.testing.expectEqual(@as(u16, 0), v); + for (got[(row * taps + 2) * c.hidden ..][0..c.hidden]) |v| try std.testing.expectEqual(@as(u16, 0), v); + } +} + +test "lfm2 dspark: a Markov step picks the argmax of bf16(base + bf16(W2 . W1[prev])) over every threadgroup" { + var r = try Rig.init(); + defer r.deinit(); + const rank = 256; + const vocab = 1300; // three scan threadgroups, the last one short + const streams = 4; + const groups = (vocab + 511) / 512; + const w1 = try r.buffer(u16, vocab * rank); + defer w1.deinit(); + noise(11, w1.slice(u16, vocab * rank), 1); + const codes = try r.buffer(u8, vocab * rank); + defer codes.deinit(); + for (codes.slice(u8, vocab * rank), 0..) |*q, i| q.* = @truncate(i *% 2654435761 >> 7); + const sc = try r.buffer(u16, vocab * rank / 64); + defer sc.deinit(); + const bi = try r.buffer(u16, vocab * rank / 64); + defer bi.deinit(); + noise(12, sc.slice(u16, vocab * rank / 64), 0.01); + noise(13, bi.slice(u16, vocab * rank / 64), 0.5); + // a token every second word (the picks' layout): inside W1, its last row, past it (a zero row), and a NaN row + const ids = try r.buffer(u32, 2 * streams); + defer ids.deinit(); + @memcpy(ids.slice(u32, 2 * streams), &[_]u32{ 17, 9, vocab - 1, 9, vocab + 5, 0, 3, 3 }); + const logits = try r.buffer(u16, streams * 2 * vocab); + defer logits.deinit(); + noise(14, logits.slice(u16, streams * 2 * vocab), 4); + logits.slice(u16, streams * 2 * vocab)[3 * 2 * vocab + 700] = 0x7fc0; + const best = try r.buffer(f32, streams * groups); + defer best.deinit(); + const at = try r.buffer(u32, streams * groups); + defer at.deinit(); + const picks = try r.buffer(u32, 2 * streams); + defer picks.deinit(); + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get("lfm2_markov_scan")); + for ([_]mtl.Buffer{ logits, w1, codes, ids }, 0..) |b, i| e.setBuffer(b, 0, i); + e.setValue([4]u32{ 2 * vocab, rank, vocab, 2 }, 4); + for ([_]mtl.Buffer{ best, at, sc, bi }, 5..) |b, i| e.setBuffer(b, 0, i); + e.dispatchGroups(mtl.Size.of(groups, streams, 1), mtl.Size.of(256, 1, 1)); + e.setPipeline(r.k.get("lfm2_markov_pick")); + e.setBuffer(best, 0, 0); + e.setBuffer(at, 0, 1); + e.setValue(@as(u32, groups), 2); + e.setBuffer(picks, 0, 3); + e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + const got = picks.slice(u32, 2 * streams); + const base = logits.slice(u16, streams * 2 * vocab); + const id_list = ids.slice(u32, 2 * streams); + for (0..streams) |s| { + if (s == 3) { // the planted NaN: flagged + try std.testing.expectEqual(@as(u32, 1), got[2 * s + 1]); + continue; + } + try std.testing.expectEqual(@as(u32, 0), got[2 * s + 1]); + const id = id_list[2 * s]; + var top: f64 = -std.math.inf(f64); + var mine: f64 = 0; + for (0..vocab) |v| { + var dot: f64 = 0; + for (0..rank) |i| { + const g = (v * rank + i) / 64; + const w = @as(f64, quant.toF32(sc.slice(u16, vocab * rank / 64)[g])) * @as(f64, @floatFromInt(codes.slice(u8, vocab * rank)[v * rank + i])) + quant.toF32(bi.slice(u16, vocab * rank / 64)[g]); + dot += w * (if (id < vocab) quant.toF32(w1.slice(u16, vocab * rank)[id * rank + i]) else 0); + } + const score: f64 = quant.toF32(quant.toBf16(quant.toF32(base[s * 2 * vocab + v]) + quant.toF32(quant.toBf16(@floatCast(dot))))); + top = @max(top, score); + if (v == got[2 * s]) mine = score; + } + // the pick is the best score, or within two bf16 steps of it where fp32 sums round apart + try std.testing.expect(got[2 * s] < vocab and mine >= top - @abs(top) * 0x1p-7); + } +} + +test "lfm2 dspark: lfm2_qk on half-split rows and gains gives interleaved RoPE's keys, reordered" { + var r = try Rig.init(); + defer r.deinit(); + const hd = c.head_dim; + const pos: u32 = 777; + const theta: f32 = 1e7; + var k_il: [c.kv_heads * hd]u16 = undefined; + var gain_il: [hd]u16 = undefined; + noise(21, &k_il, 2); + noise(22, &gain_il, 1.5); + const q = try r.buffer(u16, c.query_heads * hd); + defer q.deinit(); + const k = try r.buffer(u16, c.kv_heads * hd); + defer k.deinit(); + const v = try r.buffer(u16, c.kv_heads * hd); + defer v.deinit(); + const qn = try r.buffer(u16, hd); + defer qn.deinit(); + const kn = try r.buffer(u16, hd); + defer kn.deinit(); + for (k.slice(u16, c.kv_heads * hd), 0..) |*x, i| x.* = k_il[quant.halfSplit(i, hd)]; + for (kn.slice(u16, hd), 0..) |*x, i| x.* = gain_il[quant.halfSplit(i, hd)]; + @memset(qn.slice(u16, hd), quant.toBf16(1)); + const cap = 1024; + const queries = try r.buffer(u16, c.query_heads * hd); + defer queries.deinit(); + const keys = try r.buffer(u16, c.kv_heads * cap * hd); + defer keys.deinit(); + const values = try r.buffer(u16, c.kv_heads * cap * hd); + defer values.deinit(); + const Args = extern struct { len: u32, cap: u32, eps: f32, log2_theta: f32 }; + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get("lfm2_qk")); + for ([_]mtl.Buffer{ q, k, v, qn, kn }, 0..) |b, i| e.setBuffer(b, 0, i); + const args = Args{ .len = pos, .cap = cap, .eps = c.eps, .log2_theta = std.math.log2(theta) }; + e.setBytes(std.mem.asBytes(&args), 5); + for ([_]mtl.Buffer{ queries, keys, values }, 6..) |b, i| e.setBuffer(b, 0, i); + e.dispatchGroups(mtl.Size.of(c.query_heads + c.kv_heads, 1, 1), mtl.Size.of(32, 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + const got = keys.slice(u16, c.kv_heads * cap * hd); + for (0..c.kv_heads) |h| { + var ss: f32 = 0; + for (0..hd) |i| ss += quant.toF32(k_il[h * hd + i]) * quant.toF32(k_il[h * hd + i]); + const inv = 1 / @sqrt(ss / hd + c.eps); + for (0..hd / 2) |i| { + const angle = @as(f32, pos) * std.math.pow(f32, theta, -@as(f32, @floatFromInt(i)) / (hd / 2)); + const a = quant.toF32(k_il[h * hd + 2 * i]) * inv * quant.toF32(gain_il[2 * i]); + const b = quant.toF32(k_il[h * hd + 2 * i + 1]) * inv * quant.toF32(gain_il[2 * i + 1]); + const want = [2]f32{ a * @cos(angle) - b * @sin(angle), a * @sin(angle) + b * @cos(angle) }; + const at = (h * cap + pos) * hd; + const have = [2]f32{ quant.toF32(got[at + i]), quant.toF32(got[at + i + hd / 2]) }; + for (want, have) |w, x| try std.testing.expect(@abs(w - x) <= 0.02 * @max(1, @abs(w))); + } + } +} +test "lfm2 dspark: a settled hold moves the timed passes at its block counts a tenth of the way to its GPU ms" { + var d: ds.Drafter = undefined; + const grid = [_]lanes.hold_price.HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1 }, .{ .streams = 4, .depth = 1, .ms = 2 }, .{ .streams = 4, .depth = 4, .ms = 4 }, .{ .streams = 8, .depth = 1, .ms = 3 } }; + @memcpy(d.holds[0..grid.len], &grid); + d.holds_len = grid.len; + d.pass_len = 4; + d.depths[0..4].* = .{ 1, 1, 1, 1 }; + d.priceHold(4); // priced 2 ms: the 4-block passes rise by a tenth of the doubling + try std.testing.expectApproxEqAbs(@as(f64, 2.2), d.holds[1].ms, 1e-12); + try std.testing.expectApproxEqAbs(@as(f64, 4.4), d.holds[2].ms, 1e-12); + try std.testing.expectEqual(@as(f64, 1), d.holds[0].ms); + try std.testing.expectEqual(@as(f64, 3), d.holds[3].ms); + d.pass_len = 6; // between 4 and 8 blocks: both move + d.depths[0..6].* = .{ 1, 1, 1, 1, 1, 1 }; + d.priceHold(0); + try std.testing.expectEqual(@as(f64, 2.2), d.holds[1].ms); // no GPU time: nothing learned + d.priceHold(2.6 * 2); + try std.testing.expect(d.holds[1].ms > 2.2 and d.holds[3].ms > 3 and d.holds[0].ms == 1); +} + +/// lfm2_draft_ids on one pick (token, nonfinite flag) at a stride of 4 words: the id it wrote and the bad flag. +fn draftIds(r: *Rig, pick: Pick) ![2]u32 { + const picks = try r.buffer(u32, 4); + defer picks.deinit(); + const ids = try r.buffer(u32, 1); + defer ids.deinit(); + const bad = try r.buffer(u32, 1); + defer bad.deinit(); + picks.slice(u32, 4)[0] = pick.token; + picks.slice(u32, 4)[1] = pick.nonfinite; + try r.run("lfm2_draft_ids", &.{ picks, ids, bad }, [4]u32{ 4, 1, c.vocab, 0 }, 1, .{ 1, 1 }); + return .{ ids.slice(u32, 1)[0], bad.slice(u32, 1)[0] }; +} + +test "lfm2 dspark: held ids carry a flagged or invalid pick as token 0 and the bad flag, a good pick untouched" { + var r = try Rig.init(); + defer r.deinit(); + try std.testing.expectEqual([2]u32{ 5, 0 }, try draftIds(&r, .{ .token = 5 })); + try std.testing.expectEqual([2]u32{ c.vocab - 1, 0 }, try draftIds(&r, .{ .token = c.vocab - 1 })); + for ([_]u32{ 1, 2, 4, 7 }) |kind| try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = 9, .nonfinite = kind })); + try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = std.math.maxInt(u32) })); + try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = c.vocab })); +} + +test "lfm2 dspark: a settled hold refuses a NaN, infinity or invalid pick as the synchronous path does" { + var picks: [4 * ds.pass_rows]Pick = @splat(.{ .token = 3 }); + var a: [4]u32 = undefined; + var b: [4]u32 = undefined; + const chains = [_][]u32{ &a, &b }; + try block.copyPicks(&picks, &chains, &.{ 2, 2 }); + try std.testing.expectEqualSlices(u32, &.{ 3, 3 }, a[0..2]); + const bad = [_]Pick{ .{ .token = 3, .nonfinite = 1 }, .{ .token = 3, .nonfinite = 2 }, .{ .token = 3, .nonfinite = 4 }, .{ .token = std.math.maxInt(u32) }, .{ .token = c.vocab } }; + for (bad) |p| { + picks[ds.pass_rows + 1] = p; // the second block's second step + try std.testing.expectError(error.DraftNonfinite, block.copyPicks(&picks, &chains, &.{ 2, 2 })); + picks[ds.pass_rows + 1] = .{ .token = 3 }; + } + picks[3 * ds.pass_rows] = .{ .nonfinite = 1 }; // a step past the block's depth is not read + try block.copyPicks(&picks, &chains, &.{ 2, 2 }); +} diff --git a/zig/src/families/lfm2_moe/dspark_weights.zig b/zig/src/families/lfm2_moe/dspark_weights.zig new file mode 100644 index 0000000000..9d87dd6eed --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_weights.zig @@ -0,0 +1,178 @@ +//! The DSpark drafter's weights: the released bf16 linears as stored, or on request quantized to 8-bit affine codes. +const std = @import("std"); +const mtl = @import("metal"); +const c = @import("config.zig"); +const dc = @import("dspark_config.zig"); +const quant = @import("dspark_quant.zig"); +const ckpt = @import("../../core/checkpoint_metal.zig"); +const shards = @import("../../core/checkpoint.zig"); +const tile = @import("../../core/tile_qmm.zig"); + +const shared = mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked; + +/// An [n, k] bf16 matrix as released (rows only reordered for half-split RoPE where the checkpoint is read). +pub const Bf16 = struct { w: mtl.Buffer, n: usize, k: usize }; + +/// A linear: bf16 as stored (bits 0), or MLX affine codes at the loader's width. +pub const Linear = union(enum) { bf16: Bf16, codes: tile.Matrix }; + +/// A bf16 vector in its own buffer (norm gains; RoPE-paired ones in the half-split order the kernels rotate). +pub const Vector = mtl.Buffer; + +pub const Layer = struct { + q: Linear, + k: Linear, + v: Linear, + o: Linear, + gate: Linear, + up: Linear, + down: Linear, + input_norm: Vector, + post_norm: Vector, + q_norm: Vector, + k_norm: Vector, +}; + +pub const Weights = struct { + fc: Linear, + hidden_norm: Vector, + norm: Vector, + layers: [8]Layer, + /// The Markov head: W1 [vocab, rank] bf16 as stored (rows gathered by token), W2 [vocab, rank] as the linears. + w1: mtl.Buffer, + w2: Linear, + /// The confidence head: its [hidden + rank] bf16 weights and fp32 bias (null: the checkpoint has none). + confidence: ?mtl.Buffer = null, + confidence_bias: f32 = 0, + owned: std.ArrayList(mtl.Buffer) = .empty, + + pub fn deinit(w: *Weights, gpa: std.mem.Allocator) void { + for (w.owned.items) |b| b.deinit(); + w.owned.deinit(gpa); + } + + /// Every buffer the weights hold, for a residency set. + pub fn buffers(w: *const Weights) []const mtl.Buffer { + return w.owned.items; + } +}; + +const Loader = struct { + gpa: std.mem.Allocator, + device: mtl.Device, + check: *const ckpt.Checkpoint, + stage: *ckpt.Stage, // a matrix read from the file, one at a time, into one reused buffer + owned: *std.ArrayList(mtl.Buffer), + bits: u8, + + fn buffer(l: Loader, bytes: usize) !mtl.Buffer { + const b = try l.device.buffer(bytes, shared); + errdefer b.deinit(); + try l.owned.append(l.gpa, b); + return b; + } + + /// A vector as loaded, or a matrix read into the stage (valid until the next matrix), of `shape` in bf16. + fn tensor(l: Loader, name: []const u8, shape: []const usize) !ckpt.Tensor { + l.stage.reset(); + if (shape.len == 2) try l.stage.reserve(l.check, &.{name}); + const t = if (shape.len == 2) try l.stage.read(l.check, name) else try l.check.get(name); + if (t.dtype != .bf16 or t.rank != shape.len or !std.mem.eql(usize, t.shape[0..t.rank], shape)) { + std.log.err("{s}: unexpected DSpark tensor dtype or shape", .{name}); + return error.BadDraftWeight; + } + return t; + } + + /// A bf16 vector copied out, its pairs reordered for half-split RoPE when `paired`. + fn vector(l: Loader, name: []const u8, len: usize, paired: bool) !Vector { + const src = (try l.tensor(name, &.{len})).host(u16); + const b = try l.buffer(len * 2); + const dst = b.slice(u16, len); + for (dst, 0..) |*v, i| v.* = src[if (paired) quant.halfSplit(i, len) else i]; + return b; + } + + /// An [n, k] bf16 matrix as it is or as affine codes; rows reordered for half-split RoPE when `paired`. + fn matrix(l: Loader, name: []const u8, n: usize, k: usize, paired: bool) !Linear { + const t = try l.tensor(name, &.{ n, k }); + var rows = t.host(u16); + var moved: ?[]u16 = null; + defer if (moved) |m| l.gpa.free(m); + if (paired) { + const m = try l.gpa.alloc(u16, n * k); + moved = m; + for (0..n) |r| @memcpy(m[r * k ..][0..k], rows[quant.halfSplit(r, c.head_dim) * k ..][0..k]); + rows = m; + } + if (l.bits == 0) { + const b = try l.buffer(n * k * 2); + @memcpy(b.slice(u16, n * k), rows); + return .{ .bf16 = .{ .w = b, .n = n, .k = k } }; + } + const codes = try l.buffer(n * k * l.bits / 8); + const scales = try l.buffer(n * k / 64 * 2); + const biases = try l.buffer(n * k / 64 * 2); + const job: quant.Job = .{ .src = rows, .k = k, .bits = l.bits, .codes = codes.slice(u8, n * k * l.bits / 8), .scales = scales.slice(u16, n * k / 64), .biases = biases.slice(u16, n * k / 64) }; + try job.all(l.gpa, n); + return .{ .codes = .{ .w = codes, .scales = scales, .biases = biases, .n = n, .k = k, .bits = l.bits } }; + } +}; + +fn part(buf: []u8, layer: usize, name: []const u8) ![]const u8 { + return std.fmt.bufPrint(buf, "layers.{d}.{s}.weight", .{ layer, name }); +} + +/// Matrices stay in the file until the loader reads them; vectors load with the header. +fn matrices(_: []const u8, rank: usize) bool { + return rank == 2; +} + +/// The checkpoint read as released (`bits` 0) or quantized to 8-bit codes, a matrix at a time; buffers freed after. +pub fn load(gpa: std.mem.Allocator, device: mtl.Device, io: std.Io, dir: []const u8, cfg: *const dc.Config, bits: u8) !Weights { + if (bits != 0 and bits != 8) return error.UnsupportedDraftBits; + var check = ckpt.Checkpoint.init(gpa); + defer check.deinit(); + var stage: ckpt.Stage = .{ .device = device }; + defer stage.deinit(); + const files = try shards.shardFiles(gpa, io, dir); + defer shards.freeShardFiles(gpa, files); + for (files) |path| try check.addFileSplit(device, path, "", null, matrices); + var w: Weights = undefined; + w.owned = .empty; + errdefer w.deinit(gpa); + const l: Loader = .{ .gpa = gpa, .device = device, .check = &check, .stage = &stage, .owned = &w.owned, .bits = bits }; + const qw = c.query_heads * c.head_dim; + const kw = c.kv_heads * c.head_dim; + w.fc = try l.matrix("fc.weight", c.hidden, cfg.tap_count * c.hidden, false); + w.hidden_norm = try l.vector("hidden_norm.weight", c.hidden, false); + w.norm = try l.vector("norm.weight", c.hidden, false); + var name: [128]u8 = undefined; + for (w.layers[0..cfg.layers], 0..) |*ly, i| { + ly.q = try l.matrix(try part(&name, i, "self_attn.q_proj"), qw, c.hidden, true); + ly.k = try l.matrix(try part(&name, i, "self_attn.k_proj"), kw, c.hidden, true); + ly.v = try l.matrix(try part(&name, i, "self_attn.v_proj"), kw, c.hidden, false); + ly.o = try l.matrix(try part(&name, i, "self_attn.o_proj"), c.hidden, qw, false); + ly.gate = try l.matrix(try part(&name, i, "mlp.gate_proj"), cfg.intermediate, c.hidden, false); + ly.up = try l.matrix(try part(&name, i, "mlp.up_proj"), cfg.intermediate, c.hidden, false); + ly.down = try l.matrix(try part(&name, i, "mlp.down_proj"), c.hidden, cfg.intermediate, false); + ly.input_norm = try l.vector(try part(&name, i, "input_layernorm"), c.hidden, false); + ly.post_norm = try l.vector(try part(&name, i, "post_attention_layernorm"), c.hidden, false); + ly.q_norm = try l.vector(try part(&name, i, "self_attn.q_norm"), c.head_dim, true); + ly.k_norm = try l.vector(try part(&name, i, "self_attn.k_norm"), c.head_dim, true); + } + const w1 = (try l.tensor("markov_head.markov_w1.weight", &.{ c.vocab, cfg.rank })).host(u16); + w.w1 = try l.buffer(c.vocab * cfg.rank * 2); + @memcpy(w.w1.slice(u16, c.vocab * cfg.rank), w1); + w.w2 = try l.matrix("markov_head.markov_w2.weight", c.vocab, cfg.rank, false); + w.confidence = null; + w.confidence_bias = 0; + if (check.has("confidence_head.proj.weight")) { + const width = c.hidden + cfg.rank; // the final norm's row, then W1 of the token before the draft + const proj = (try l.tensor("confidence_head.proj.weight", &.{ 1, width })).host(u16); + w.confidence = try l.buffer(width * 2); + @memcpy(w.confidence.?.slice(u16, width), proj); + w.confidence_bias = quant.toF32((try l.tensor("confidence_head.proj.bias", &.{1})).host(u16)[0]); + } + return w; +} diff --git a/zig/src/families/lfm2_moe/forward.zig b/zig/src/families/lfm2_moe/forward.zig index 212b5978bc..51408e2fd4 100644 --- a/zig/src/families/lfm2_moe/forward.zig +++ b/zig/src/families/lfm2_moe/forward.zig @@ -118,7 +118,17 @@ pub fn matrix(w: row.Weights) tile.Matrix { return .{ .w = w.w, .w_off = w.w_off, .scales = w.scales, .s_off = w.s_off, .biases = w.biases, .b_off = w.b_off, .n = w.n, .k = w.k, .bits = w.bits }; } +/// A window's held drafts on the device: `count` tokens at `src` + `off`, `stride` words apart, into ids from `row`. +pub const Held = struct { row: usize, src: Buffer, off: usize, stride: u32, count: u32 }; +/// A GPU event the forward waits on before it reads held drafts, and the value it waits for. +pub const Wait = struct { event: mtl.Event, value: u64 }; + pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const u32, record: bool, head: Head) !void { + return runHeld(m, s, segments, tokens, record, head, &.{}, null); +} + +/// `run` with held drafts copied into the rows' ids on the GPU first (after `wait`): their host ids are ignored. +pub fn runHeld(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const u32, record: bool, head: Head, held: []const Held, wait: ?Wait) !void { if (tokens.len == 0 or tokens.len > s.rows or (record and tokens.len > st.batch_rows)) return error.LfmRowsExceeded; var total: usize = 0; for (segments) |seg| { @@ -131,10 +141,20 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const const pool = mtl.objc.Pool.push(); defer pool.pop(); const cb = m.queue.commandBuffer(); + if (wait) |w| cb.waitForEvent(w.event, w.value); const prompt = (!record and head == .last) or total >= tile_rows; const e = cb.compute(.serial); const plan = if (segs.batched(record, segments.len)) segs.prepare(e, s, segments) else null; const enc = Encoder{ .e = e, .m = m, .s = s, .prompt = prompt, .record = record, .plan = plan }; + if (held.len > 0) @memset(s.bad.contents()[0..4], 0); + for (held) |h| { + enc.pipe("lfm2_draft_ids"); + e.setBuffer(h.src, h.off, 0); + e.setValue([4]u32{ h.stride, h.count, c.vocab, 0 }, 1); + e.setBuffer(s.ids, h.row * 4, 2); + e.setBuffer(s.bad, 0, 3); + enc.threads(.{ h.count, 1, 1 }, .{ 32, 1, 1 }); + } const embedding = m.weights.embedding; e.setPipeline(m.kernels.embed(embedding.bits)); e.setBuffer(s.ids, 0, 0); @@ -146,6 +166,7 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const for (m.weights.blocks, 0..) |block, i| { enc.norm(s, block.operator_norm, i == 0, total, combine); combine = false; + if (i > 0) tap(enc, s, i - 1, total); switch (block.mixer) { .conv => |cv| conv(enc, s, cv, i, segments, total), .attention => |a| attention(enc, s, a, &m.config, segments, i, total), @@ -162,6 +183,7 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const } } enc.norm(s, m.weights.norm, false, total, combine); + tap(enc, s, c.layers - 1, total); if (head == .all) { enc.projection(embedding, s.x, 0, s.logits, 0, total); enc.pipe("lfm2_argmax"); @@ -178,6 +200,7 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const std.debug.print("LFM2 forward: {s}\n", .{msg}); return error.GpuFailed; } + if (held.len > 0 and std.mem.readInt(u32, s.bad.contents()[0..4], .little) != 0) return error.DraftNonfinite; var base: usize = 0; for (segments) |seg| { const logit_row = if (head == .all) base + seg.rows - 1 else 0; @@ -187,6 +210,18 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const } } +/// Layer `layer`'s output rows (h, just summed by the next norm) into each tap of it, when a drafter reads them. +fn tap(enc: Encoder, s: *st.Scratch, layer: usize, total: usize) void { + const taps = s.taps orelse return; + for (taps.layers[0..taps.count], 0..) |l, t| if (l == layer) { + enc.pipe("lfm2_tap"); + enc.e.setBuffer(s.h, 0, 0); + enc.e.setBuffer(taps.buf, 0, 1); + enc.e.setValue([2]u32{ taps.count * c.hidden, @intCast(t * c.hidden) }, 2); + enc.threads(.{ c.hidden / 8, total, 1 }, .{ 256, 1, 1 }); + }; +} + fn conv(enc: Encoder, s: *st.Scratch, cv: wts.Conv, layer: usize, segments: []const Segment, total: usize) void { const e = enc.e; enc.projection(cv.in, s.x, 0, s.bcx, 0, total); diff --git a/zig/src/families/lfm2_moe/kernels.zig b/zig/src/families/lfm2_moe/kernels.zig index aa1e798fc0..c83c9a0b4a 100644 --- a/zig/src/families/lfm2_moe/kernels.zig +++ b/zig/src/families/lfm2_moe/kernels.zig @@ -16,6 +16,8 @@ const names = [_][:0]const u8{ "lfm2_attn_mma", "lfm2_attn_mma_r2", "lfm2_attn_merge", + "lfm2_attn_block", + "lfm2_attn_block_merge", "lfm2_route_logits", "lfm2_route_select", "lfm2_swiglu", @@ -27,6 +29,14 @@ const names = [_][:0]const u8{ "lfm2_qk_segs", "lfm2_attn_segs", "lfm2_attn_merge_segs", + "lfm2_tap", + "lfm2_markov_scan", + "lfm2_markov_scan_bf16", + "lfm2_bf16_rows", + "lfm2_bf16_mma", + "lfm2_markov_pick", + "lfm2_draft_ids", + "lfm2_dspark_confidence", }; pub const Kernels = struct { diff --git a/zig/src/families/lfm2_moe/lfm2_moe.zig b/zig/src/families/lfm2_moe/lfm2_moe.zig index 97f9cbd5d8..8f861e31b4 100644 --- a/zig/src/families/lfm2_moe/lfm2_moe.zig +++ b/zig/src/families/lfm2_moe/lfm2_moe.zig @@ -7,3 +7,7 @@ pub const state = @import("state.zig"); pub const forward = @import("forward.zig"); pub const backend = @import("backend.zig"); pub const snapshot = @import("snapshot.zig"); +pub const dspark = @import("dspark.zig"); +pub const dspark_config = @import("dspark_config.zig"); +pub const dspark_block = @import("dspark_block.zig"); +pub const dspark_quant = @import("dspark_quant.zig"); diff --git a/zig/src/families/lfm2_moe/snapshot.zig b/zig/src/families/lfm2_moe/snapshot.zig index d1c1a8b3ae..d7289331b7 100644 --- a/zig/src/families/lfm2_moe/snapshot.zig +++ b/zig/src/families/lfm2_moe/snapshot.zig @@ -4,7 +4,7 @@ const c = @import("config.zig"); const st = @import("state.zig"); /// One state: conv states by conv layer, then keys and values [kv head][at][head dim] by attention layer. -pub const Snap = struct { at: u32, bytes: []u8 }; +pub const Snap = struct { at: u32, bytes: []u8, draft: ?[]u8 = null }; /// A row's keys or values for every kv head of one attention layer. const row_bytes = c.kv_heads * c.head_dim * 2; @@ -56,6 +56,7 @@ pub fn restore(cache: *st.Cache, snap: *const Snap) !void { } pub fn drop(gpa: std.mem.Allocator, snap: *Snap) void { + if (snap.draft) |d| gpa.free(d); // the drafter's context at `at`, when one was kept with the state gpa.free(snap.bytes); gpa.destroy(snap); } diff --git a/zig/src/families/lfm2_moe/state.zig b/zig/src/families/lfm2_moe/state.zig index 3242d8bf26..883f670fbe 100644 --- a/zig/src/families/lfm2_moe/state.zig +++ b/zig/src/families/lfm2_moe/state.zig @@ -51,6 +51,12 @@ pub const Storage = struct { }; pub const Attention = struct { keys: mtl.Buffer, values: mtl.Buffer }; +/// Layers a drafter reads at most (each layer's output, hidden wide). +pub const max_taps = 8; +/// The drafter's layers in its order and every forward row's outputs of them, [rows][count][hidden] bf16. +pub const Taps = struct { layers: [max_taps]u32 = @splat(0), count: u32 = 0, buf: mtl.Buffer }; +/// Where a stream's latest tapped rows sit in the scratch: cache rows [start, start + rows) from scratch row `row`. +pub const TapRows = struct { start: usize, row: usize, rows: usize, gen: u64 }; pub const Mixer = union(enum) { conv: mtl.Buffer, attention: Attention }; pub const Cache = struct { @@ -60,6 +66,9 @@ pub const Cache = struct { last: ?struct { start: usize, base: usize, rows: usize } = null, blocks: [c.layers]Mixer, logits: mtl.Buffer, + /// The stream's latest tapped rows in the scratch, and its prompt's last row's taps (kept for the first draft). + tap: ?TapRows = null, + last_tap: ?mtl.Buffer = null, pub fn init(gpa: std.mem.Allocator, device: mtl.Device, attention: *const [c.layers]bool, capacity: usize) !Cache { var out: Cache = .{ .memory = .{ .gpa = gpa, .device = device }, .capacity = capacity, .blocks = undefined, .logits = undefined }; @@ -110,6 +119,7 @@ pub const Scratch = struct { memory: Storage, rows: usize, ids: mtl.Buffer, + bad: mtl.Buffer, // set by lfm2_draft_ids when a held pick is flagged nonfinite or past the vocabulary h: mtl.Buffer, x: mtl.Buffer, r: mtl.Buffer, @@ -148,6 +158,8 @@ pub const Scratch = struct { segs: mtl.Buffer, rowseg: mtl.Buffer, tiles: mtl.Buffer, + /// A drafter's taps of every row, once the backend is told its layers (null: none kept). + taps: ?Taps = null, pub fn init(gpa: std.mem.Allocator, device: mtl.Device, attention: *const [c.layers]bool, rows: usize) !Scratch { var out: Scratch = undefined; @@ -155,9 +167,11 @@ pub const Scratch = struct { errdefer out.memory.deinit(); out.rows = rows; out.snapshots = @splat(null); + out.taps = null; const mem = &out.memory; const slots = rows * c.topk; out.ids = try mem.alloc(rows * 4); + out.bad = try mem.alloc(4); inline for (.{ "h", "x", "r", "y", "q" }) |name| @field(out, name) = try mem.alloc(rows * c.hidden * 2); out.queries = try mem.alloc((rows + attn_rows) * c.hidden * 2); // a prompt attention tile reads attn_rows from its first out.bcx = try mem.alloc(rows * 3 * c.hidden * 2); diff --git a/zig/src/native/cuda_options.zig b/zig/src/native/cuda_options.zig index d6a6eefe28..0c07152bdb 100644 --- a/zig/src/native/cuda_options.zig +++ b/zig/src/native/cuda_options.zig @@ -36,6 +36,10 @@ pub fn set(comptime F: type, o: api.Open, opts: *F.Options, problem: *[]const u8 return false; }; } + if (o.drafter_bits_given and o.drafter_bits == 8) { + problem.* = "--drafter-bits 8: only LFM2.5-24B-A2B's Metal DSpark drafter takes it; this family takes 0 (bf16) or 4"; + return false; + } if (comptime @hasField(F.Options, "drafter")) opts.drafter = o.drafter; if (comptime @hasField(F.Options, "drafter_bits")) opts.drafter_bits = o.drafter_bits; if (@hasField(F.Options, "kv8")) opts.kv8 = o.kv8; diff --git a/zig/src/native/draft_sources_metal_test.zig b/zig/src/native/draft_sources_metal_test.zig index 2bd5851477..0c295d80f4 100644 --- a/zig/src/native/draft_sources_metal_test.zig +++ b/zig/src/native/draft_sources_metal_test.zig @@ -20,6 +20,7 @@ test "draft sources: Metal Qwen2B and LFM2 MoE follow qualified verification wid q.options.streams = 2; l.options.streams = 2; l.timed = 0; + l.shared_timed = 0; for ([_]u32{ 1, 16 }) |width| { q.width = width; l.width = width; diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index 5340d807de..3734c41a3c 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -1,4 +1,4 @@ -//! LFM2-MoE served by the existing lane host and scheduler. +//! LFM2-MoE served by the existing lane host and scheduler, with LiquidAI's DSpark drafter from --drafter. const std = @import("std"); const mtl = @import("metal"); const api = @import("engine_api"); @@ -8,11 +8,16 @@ const lanes = tf.lanes; const cache_fit = @import("cache_fit.zig"); const metal_admit = @import("metal_admit.zig"); +/// --drafter loads LiquidAI's DSpark drafter (LiquidAI/LFM2.5-24B-A2B-DSpark) in front of the engine. +pub const takes_drafter = true; + const Host = struct { resident: ?mtl.resident.Resident = null, gpa: std.mem.Allocator, model: *q.Model, metal: *q.backend.Metal, + draft: ?*q.dspark.Drafter = null, + wrap: ?lanes.drafted.Drafted = null, warm: mtl.keepalive.Target = undefined, // the model's queue, for the lane host's idle ticker config: lanes.Config, clock: lanes.backend.WallClock, @@ -25,7 +30,9 @@ const Host = struct { h.host.stop(); if (h.cache) |*store| store.deinit(); h.core.deinit(); + if (h.wrap) |*w| w.deinit(); mtl.resident.streams.drop(h); + if (h.draft) |d| d.deinit(); h.config.deinit(h.gpa); h.metal.deinit(); h.model.deinit(); @@ -35,6 +42,13 @@ const Host = struct { const cannot = "the native LFM2 engine cannot load this checkpoint ({s})"; +/// A refused open's backend, model and host, freed before it reports why. +fn release(h: *Host) void { + h.metal.deinit(); + h.model.deinit(); + h.gpa.destroy(h); +} + pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Open, problem: *[]const u8) !?api.Opened { const pool = mtl.objc.Pool.push(); defer pool.pop(); @@ -65,18 +79,49 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope const chunk = 2048; h.metal = try q.backend.Metal.init(gpa, model, .{ .capacity = window, .chunk = chunk, .streams = @max(o.lanes, 2) }); errdefer h.metal.deinit(); - const per_stream = q.state.cacheBytes(&model.config.attention, window); + h.draft = null; + h.wrap = null; + if (if (o.drafts) o.drafter else null) |dir| { + if (h.metal.width == 1) { + problem.* = "--drafter: LFM2 wider lanes failed the local exactness check; drafts require verified wider lanes"; + release(h); + return null; + } + const bits: u8 = if (o.drafter_bits_given) o.drafter_bits else 0; + if (bits != 0 and bits != 8) { + problem.* = "--drafter-bits: the LFM2 DSpark drafter takes 0 (bf16 as released, the default) or 8 (8-bit codes quantized at load)"; + release(h); + return null; + } + h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window, .bits = bits }) catch |err| { + problem.* = try std.fmt.allocPrint(a, "the native LFM2 drafter cannot load {s} ({s}): it takes LiquidAI/LFM2.5-24B-A2B-DSpark", .{ dir, @errorName(err) }); + release(h); + return null; + }; + } + errdefer if (h.draft) |d| d.deinit(); + // a stream's caches, and its drafter lane when one drafts + const per_stream = q.state.cacheBytes(&model.config.attention, window) + if (h.draft) |d| d.laneBytes(window) else 0; const served = try metal_admit.streams(a, "lfm2", per_stream, metal_admit.room(model.device), o, window, problem) orelse { - h.metal.deinit(); - model.deinit(); - gpa.destroy(h); + if (h.draft) |d| d.deinit(); + release(h); return null; }; h.metal.options.streams = @max(served, 2); - h.config = try lanes.Config.init(gpa, h.metal.facts(), q.state.window_rows, q.state.window_rows - 1); + if (h.draft) |d| { + h.wrap = try lanes.drafted.Drafted.init(gpa, h.metal.backend(), d.drafter()); + // the verify copies the drafts on the GPU after the unwaited hold + h.metal.held_source = d.heldSource(); + h.wrap.?.held_on_target = true; + } + errdefer if (h.wrap) |*w| w.deinit(); + var facts = if (h.wrap) |*w| w.facts(h.metal.facts()) else h.metal.facts(); + // a drafting stream prices its copy windows by its own copies: on distinct prompts they keep breaking + if (h.wrap != null) facts.own_copy_rates = true; + h.config = try lanes.Config.init(gpa, facts, q.state.window_rows, q.state.window_rows - 1); errdefer h.config.deinit(gpa); h.clock = .{ .io = io }; - h.core = lanes.Engine.init(gpa, &h.config, h.metal.backend(), h.clock.clock()); + h.core = lanes.Engine.init(gpa, &h.config, if (h.wrap) |*w| w.backend() else h.metal.backend(), h.clock.clock()); errdefer h.core.deinit(); h.host = api.LaneHost.init(gpa, io, &h.core, .{ .lanes = served, .context_window = @intCast(window), .prefill_step = chunk, .structures = true }); h.cache = null; @@ -97,8 +142,8 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope if (plan.budget_bytes > 0) { const S = q.backend.Snaps; h.cache = api.prompt_cache.Store.init(gpa, .{ - .ptr = h.metal, - .vtable = &.{ .bytes = S.bytes, .save = S.save, .restore = S.restore, .drop = S.drop }, + .ptr = h, + .vtable = &.{ .bytes = Snaps.bytes, .save = Snaps.save, .restore = S.restore, .drop = Snaps.drop }, }, .{ .chunk = chunk }, plan.budget_bytes); h.host.cache = &h.cache.?; h.host.info_.prompt_cache = true; @@ -106,7 +151,40 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope errdefer if (h.cache) |*store| store.deinit(); try mtl.resident.streams.hold(h, io, "lfm2 moe"); errdefer mtl.resident.streams.drop(h); + if (h.draft) |d| if (h.resident) |*r| { + const held = try d.buffers(gpa); + defer gpa.free(held); + r.add(held); + }; h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; try h.host.start(); + if (h.draft) |d| { + const mode = if (d.options.bits == 8) "8-bit codes quantized at load (--drafter-bits 8), not the released bf16" else "bf16 matrices as released"; + std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, {s}", .{ d.config.block, mode }); + } return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; } + +/// The prompt cache's copies of a stream: the target's state at a mark (backend.zig), and the drafter's context there. +const Snaps = struct { + const S = q.backend.Snaps; + fn host(ptr: *anyopaque) *Host { + return @ptrCast(@alignCast(ptr)); + } + fn bytes(ptr: *anyopaque, at: u32) u64 { + const h = host(ptr); + return S.bytes(h.metal, at) + if (h.draft) |d| d.keptBytes(at) else 0; + } + fn save(ptr: *anyopaque, owner: ?*anyopaque, at: u32) anyerror!*anyopaque { + const h = host(ptr); + const saved = try S.save(h.metal, owner, at); + if (h.draft) |d| d.keepLane(@ptrCast(@alignCast(owner.?)), @ptrCast(@alignCast(saved))) catch |err| { + S.drop(h.metal, saved); + return err; + }; + return saved; + } + fn drop(ptr: *anyopaque, saved: *anyopaque) void { + S.drop(host(ptr).metal, saved); + } +}; diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index 1391c64f86..fbad2ae996 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -176,8 +176,8 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c problem.* = why; return null; } - if (o.drafter != null and !dense and !isGemma4(o.model_type)) { - problem.* = "--drafter is supported for Qwen3.8-27B-class checkpoints (qwen3_5 with its own output head) and Gemma 4 dense"; + if (o.drafter != null and !dense and !isGemma4(o.model_type) and !std.mem.eql(u8, o.model_type, "lfm2_moe")) { + problem.* = "--drafter is supported for Qwen3.8-27B-class checkpoints (qwen3_5 with its own output head), Gemma 4 dense and LFM2.5-24B-A2B"; return null; } if (dense) return openQwen27(a, gpa, io, o, problem); diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index 87914e0042..5f26ed3472 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -100,7 +100,7 @@ pub const flags = [_]Flag{ .{ .name = "--slide", .kind = .store_true, .native = true }, .{ .name = "--slide-graph", .native = true }, .{ .name = "--drafter", .native = true }, - .{ .name = "--drafter-bits", .choices = &.{ "0", "4" }, .native = true }, + .{ .name = "--drafter-bits", .choices = &.{ "0", "4", "8" }, .native = true }, .{ .name = "--record", .native = true }, .{ .name = "--kv-cache", .choices = &.{ "bf16", "fp8" }, .native = true }, .{ .name = "--prompt-precision", .choices = &.{ "bf16", "int8" }, .native = true }, @@ -152,7 +152,6 @@ pub const flags = [_]Flag{ // the native server's own: Cortheon's reasoning program and server-run web research for every chat request .{ .name = "--cortheon", .kind = .store_true, .native = true }, }; - /// The variables this binary honours as the Python engine does, then the CUDA build's or the Metal build's. pub const env = [_][]const u8{ "TENSORFOLD_API_KEY", "TENSORFOLD_NO_LIVE", "TENSORFOLD_SEED_SALT", "TENSORFOLD_REQUEST_LOG", "TENSORFOLD_NO_UPDATE_CHECK", "HF_HOME", "HF_HUB_CACHE", "HF_HUB_OFFLINE", "BRAVE_SEARCH_API_KEY" } ++ (if (cuda_build) [_][]const u8{ "TF_CUDA_DEVICE", "TF_CUDA_SEGMENTS", "TENSORFOLD_CUDA_KERNELS", "TENSORFOLD_MEMORY_RESERVE_GIB", "TENSORFOLD_CUDA_MEMORY_LIMIT_GB", "TENSORFOLD_P2P_DIAGNOSTICS", "TENSORFOLD_P2P_PROBE" } else [_][]const u8{}) ++ @@ -217,7 +216,7 @@ pub const Args = struct { compact_memory: ?[]const u8 = null, drafter: ?[]const u8 = null, drafter_name: ?[]const u8 = null, // retained before hub resolution for the engine's serving line - drafter_bits: u8 = 4, + drafter_bits: ?u8 = null, // null: not given (a family without its own default takes 4) record: ?[]const u8 = null, // drafter training rows written under this folder kv_cache: []const u8 = "bf16", // fp8: e4m3 attention caches with a scale a row, about twice the context prompt_precision: []const u8 = "bf16", // int8: prompt matmuls quantize their inputs to int8 a 64-input group @@ -548,7 +547,8 @@ pub fn request(a: Allocator, dir: []const u8, model_type: []const u8, args: Args .drafts = !args.no_drafts, .drafter = args.drafter, .drafter_name = args.drafter_name orelse args.drafter, - .drafter_bits = args.drafter_bits, + .drafter_bits = args.drafter_bits orelse 4, + .drafter_bits_given = args.drafter_bits != null, .record = args.record, .kv8 = std.mem.eql(u8, args.kv_cache, "fp8"), .prompt_int8 = std.mem.eql(u8, args.prompt_precision, "int8"), diff --git a/zig/src/server/cli_test.zig b/zig/src/server/cli_test.zig index ca4455cf16..123e4dfe10 100644 --- a/zig/src/server/cli_test.zig +++ b/zig/src/server/cli_test.zig @@ -163,8 +163,10 @@ test "parse and capabilities share the table" { try std.testing.expectEqual(@as(usize, 2), args.api_key.len); try std.testing.expect(!args.thinking); try std.testing.expectEqualStrings("x", (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter.?); - try std.testing.expectEqual(@as(u8, 0), (try parse(a, &.{ "m", "--drafter-bits", "0" }, &u)).drafter_bits); - try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "8" }, &u)); + try std.testing.expectEqual(@as(?u8, 0), (try parse(a, &.{ "m", "--drafter-bits", "0" }, &u)).drafter_bits); + try std.testing.expectEqual(@as(?u8, 8), (try parse(a, &.{ "m", "--drafter-bits", "8" }, &u)).drafter_bits); + try std.testing.expectEqual(@as(?u8, null), (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter_bits); + try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "3" }, &u)); try std.testing.expectEqualStrings("/r", (try parse(a, &.{ "m", "--record", "/r" }, &u)).record.?); try std.testing.expectEqualStrings("fp8", (try parse(a, &.{ "m", "--kv-cache", "fp8" }, &u)).kv_cache); try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--kv-cache", "int4" }, &u)); diff --git a/zig/src/server/main.zig b/zig/src/server/main.zig index 7ab94683aa..ab48bd608a 100644 --- a/zig/src/server/main.zig +++ b/zig/src/server/main.zig @@ -26,7 +26,7 @@ const usage_line = "usage: tensorfold serve [-h] [--host HOST] [--port PORT] [-- " [--thinking | --no-thinking] [--reasoning-effort {low,medium,high,xhigh}]" ++ " [--thinking-budget THINKING_BUDGET] [--thinking-reserve THINKING_RESERVE] [--loop-guard] [--loop-guard-stop] [--no-drafts]" ++ " [--drafter DRAFTER]" ++ - " [--drafter-bits {0,4}] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++ + " [--drafter-bits {0,4,8}] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++ " [--silence-floor-dbfs DBFS] [--silence-seconds SECONDS]" ++ " [--silence-onset-s SECONDS] [--silence-onset-voiced-s SECONDS]" ++ " [--silence-mid-s SECONDS] [--silence-mid-min-voiced-s SECONDS] [--silence-redraws N]" ++ diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 4fa4fbe147..cd9d82bc08 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -88,6 +88,10 @@ test { _ = @import("families/lfm2_moe/config.zig"); _ = @import("families/lfm2_moe/backend.zig"); _ = @import("families/lfm2_moe/state.zig"); + _ = @import("families/lfm2_moe/dspark_config.zig"); + _ = @import("families/lfm2_moe/dspark_quant.zig"); + _ = @import("families/lfm2_moe/dspark_test.zig"); + _ = @import("families/lfm2_moe/dspark_bf16_test.zig"); } pub const lane_projection = @import("core/lane_projection.zig"); diff --git a/zig/tests/contract/metal_lfm2_attn.zig b/zig/tests/contract/metal_lfm2_attn.zig new file mode 100644 index 0000000000..60fa6939ae --- /dev/null +++ b/zig/tests/contract/metal_lfm2_attn.zig @@ -0,0 +1,172 @@ +//! LFM2's block attention (lfm2_attn_block, then lfm2_attn_block_merge): every row of a drafter's block sees all keys. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const c = contract.spec; +const g = contract.gates; +const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); + +const qh = 32; +const kvh = 8; +const hd = 64; +const subs = 8; // the drafter's sub-spans a 4,096-key chunk (state.subs) +const chunks = 32; +const chunk_keys = 4096; +pub const name = "metal lfm2 block attention"; + +/// rows: a block's rows; k: the keys every row sees (a context and the block past it). +const shapes = [_]c.Shape{ .{ .rows = 1, .n = qh * hd, .k = 9 }, .{ .rows = 3, .n = qh * hd, .k = 700 }, .{ .rows = 9, .n = qh * hd, .k = 4500 } }; + +fn depth(s: c.Shape) usize { + return s.k; +} + +pub fn spec() c.Spec { + return .{ .name = name, .accum = .none, .depth = depth, .out = .bf16, .served = &shapes, .nan = .{ .stated = "a NaN key of a kv head makes its query heads' outputs NaN, the others stay finite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "the check bounds each output by its bf16 probabilities' rounding, not a sum's bits" } }; +} + +fn value(i: usize, seed: u64, scale: f32) u16 { + const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40; + return c.toBf16((@as(f32, @floatFromInt(h % 2048)) / 1024 - 1) * scale); +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + mutation: controls.Mutation, + pipes: [2]mtl.Pipeline, + shape: c.Shape = undefined, + cap: usize = 0, + nan: bool = false, + q: []u16 = &.{}, + k: []u16 = &.{}, + v: []u16 = &.{}, + args: [5]rig_.Ref = undefined, + y: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + const first = try rig_.pipeline(device, text, "lfm2_attn_block"); + errdefer first.deinit(); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipes = .{ first, try rig_.pipeline(device, text, "lfm2_attn_block_merge") }, .mutation = mutation }; + } + + fn drop(p: *Probe) void { + inline for (.{ "q", "k", "v" }) |f| { + p.a.free(@field(p, f)); + @field(p, f) = &.{}; + } + p.rig.reset(); + } + + pub fn deinit(p: *Probe) void { + p.drop(); + p.rig.deinit(); + for (p.pipes) |x| x.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + var s = spec(); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() }; + } + + pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { + if (s.n != qh * hd or s.rows == 0 or s.rows > 16 or s.k < s.rows or s.k > chunks * chunk_keys) return false; + p.drop(); + p.shape = s; + p.nan = f == .nonfinite; + p.cap = (s.k + 63) / 64 * 64 + 32; // keys past `k` hold finite values, as a lane's cache does + const padded = (s.rows + 7) / 8 * 8 + 8; // the kernel reads a tile's 8 rows whatever its row count + p.q = try p.a.alloc(u16, padded * qh * hd); + p.k = try p.a.alloc(u16, kvh * p.cap * hd); + p.v = try p.a.alloc(u16, kvh * p.cap * hd); + for (p.q, 0..) |*x, i| x.* = value(i, seed, 1); + for (p.k, 0..) |*x, i| x.* = value(i, seed +% 1, if (f == .stress) 4 else 1); + for (p.v, 0..) |*x, i| x.* = value(i, seed +% 2, 2); + if (p.nan) p.k[(0 * p.cap + s.k / 2) * hd + 5] = 0x7fc0; // kv head 0, a visible key + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.q)); + p.args[1] = try p.rig.input(std.mem.sliceAsBytes(p.k)); + p.args[2] = try p.rig.input(std.mem.sliceAsBytes(p.v)); + const Args = extern struct { len: u32, cap: u32, scale: f32, subs: u32 }; + const args = Args{ .len = @intCast(s.k - 1), .cap = @intCast(p.cap), .scale = 1.0 / @sqrt(@as(f32, hd)), .subs = subs }; + p.args[3] = try p.rig.input(std.mem.asBytes(&args)); + const n: u32 = @intCast(s.rows); + p.args[4] = try p.rig.input(std.mem.asBytes(&n)); + p.y = try p.rig.output(p.count() * 2); + return true; + } + + pub fn count(p: *Probe) usize { + return p.shape.rows * qh * hd; + } + + pub fn run(p: *Probe) !void { + const rows = p.shape.rows; + const partials = @max(rows, 8) * subs * qh * chunks; + const pm = try p.rig.scratch(partials * 4); + const pl = try p.rig.scratch(partials * 4); + const po = try p.rig.scratch(partials * hd * 4); + const pass = p.rig.begin(); + const e = pass.e; + e.setPipeline(p.pipes[0]); + for (p.args[0..4], 0..) |r, i| e.setBuffer(r.buf, r.off, i); + for ([_]rig_.Ref{ pm, pl, po, p.args[4] }, 4..) |r, i| e.setBuffer(r.buf, r.off, i); + const used = (p.shape.k + chunk_keys - 1) / chunk_keys * subs; + e.dispatchGroups(mtl.Size.of((rows + 7) / 8, kvh, used), mtl.Size.of(128, 1, 1)); + e.setPipeline(p.pipes[1]); + for ([_]rig_.Ref{ pm, pl, po, p.args[3], p.y }, 0..) |r, i| e.setBuffer(r.buf, r.off, i); + e.dispatchGroups(mtl.Size.of(qh, rows, 1), mtl.Size.of(32, 1, 1)); + try p.rig.finish(pass); + } + + pub fn reference(_: *Probe, want: []f64, mag: []f64) !void { + @memset(want, 0); // the check computes its own references + @memset(mag, 0); + } + + /// Every output against float64 attention: within 2^-7 of its values' largest magnitude and 2^-8 of itself. + pub fn check(p: *Probe, _: g.Gate) !?[]const u8 { + const s = p.shape; + const out = std.mem.bytesAsSlice(u16, p.rig.bytes(p.y, p.count() * 2)); + const w = try p.a.alloc(f64, s.k); + defer p.a.free(w); + for (0..s.rows) |r| for (0..qh) |h| { + const kv = h / (qh / kvh); + const nan_head = p.nan and kv == 0; + var top: f64 = -std.math.inf(f64); + for (0..s.k) |j| { + var dot: f64 = 0; + for (0..hd) |d| dot += c.fromBf16(p.q[(r * qh + h) * hd + d]) * c.fromBf16(p.k[(kv * p.cap + j) * hd + d]); + w[j] = dot / @sqrt(@as(f64, hd)); + top = @max(top, w[j]); + } + var sum: f64 = 0; + for (w) |*x| { + x.* = @exp(x.* - top); + sum += x.*; + } + for (0..hd) |d| { + var o: f64 = 0; + var big: f64 = 0; + for (0..s.k) |j| { + const vv: f64 = c.fromBf16(p.v[(kv * p.cap + j) * hd + d]); + o += w[j] / sum * vv; + big = @max(big, @abs(vv)); + } + const got: f64 = c.fromBf16(out[(r * qh + h) * hd + d]); + if (nan_head) { + if (!std.math.isNan(got)) return "a head behind a NaN key gave a number"; + continue; + } + if (!std.math.isFinite(got)) return "a finite head gave a non-finite output"; + if (@abs(got - o) > big * 0x1p-7 + @abs(o) * 0x1p-8) return "an output is past its float64 bound"; + } + }; + return null; + } +}; diff --git a/zig/tests/contract/metal_lfm2_bf16.zig b/zig/tests/contract/metal_lfm2_bf16.zig new file mode 100644 index 0000000000..fcbe718243 --- /dev/null +++ b/zig/tests/contract/metal_lfm2_bf16.zig @@ -0,0 +1,135 @@ +//! LFM2's DSpark bf16 linears under gates 1-4: x W^T with bf16 inputs and fp32 sums, few rows or 64x64 tiles. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); + +const c = contract.spec; +const g = contract.gates; +const Dense = contract.dense.Dense; + +pub const Which = enum { rows, tile }; +const names = [_][]const u8{ "metal lfm2 bf16 rows", "metal lfm2 bf16 tile" }; + +/// The drafter's matrices: q and o (2,048 from 2,048), k and v (512), gate and up (6,144), down (2,048 from 6,144). +const rows_served = [_]c.Shape{ + .{ .rows = 1, .n = 2048, .k = 2048 }, .{ .rows = 9, .n = 512, .k = 2048 }, .{ .rows = 16, .n = 6144, .k = 2048 }, + .{ .rows = 32, .n = 2048, .k = 6144 }, .{ .rows = 4, .n = 2048, .k = 10240 }, +}; +/// Shared holds past 16 rows, and a cold prompt's absorb up to 2,048 rows a pass. +const tile_served = [_]c.Shape{ + .{ .rows = 17, .n = 2048, .k = 2048 }, .{ .rows = 18, .n = 512, .k = 2048 }, + .{ .rows = 24, .n = 6144, .k = 2048 }, .{ .rows = 32, .n = 2048, .k = 6144 }, + .{ .rows = 33, .n = 2048, .k = 2048 }, .{ .rows = 128, .n = 512, .k = 2048 }, + .{ .rows = 2048, .n = 2048, .k = 10240 }, .{ .rows = 70, .n = 6144, .k = 2048 }, +}; + +/// A lane's k / 32 fp32 fmas in turn, then the simdgroup's five-level sum. +fn rowsDepth(s: c.Shape) usize { + return s.k / 32 + 8; +} + +/// One 8-wide MMA step a k step of 8, accumulating in fp32. +fn tileDepth(s: c.Shape) usize { + return s.k / 8 + 8; +} + +pub fn spec(w: Which) c.Spec { + return switch (w) { + // rows in groups of 16, a column a simdgroup (any n), k in 256-wide lane steps of 8 + .rows => .{ .name = names[0], .accum = .f32, .depth = rowsDepth, .out = .bf16, .tiles = .{ .{ .size = 16 }, .{ .size = 8 }, .{ .size = 256, .step = 8, .min = 8 } }, .served = &rows_served }, + .tile => .{ .name = names[1], .accum = .f32, .depth = tileDepth, .out = .bf16, .tiles = .{ .{ .size = 64 }, .{ .size = 64 }, .{ .size = 32, .step = 32, .min = 32 } }, .served = &tile_served }, + }; +} + +const Args = extern struct { rows: u32, n: u32, k: u32 }; + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + which: Which, + mutation: controls.Mutation, + pipe: mtl.Pipeline, + d: ?Dense = null, + x: rig_.Ref = undefined, + w: rig_.Ref = undefined, + out: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + const key: []const u8 = if (w == .rows) "lfm2_bf16_rows" else "lfm2_bf16_mma"; + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .which = w, .mutation = mutation, .pipe = try rig_.pipeline(device, text, key) }; + } + + pub fn deinit(p: *Probe) void { + if (p.d) |d| d.deinit(); + p.rig.deinit(); + p.pipe.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + var s = spec(p.which); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .spec = s, .ctx = p, .vt = &vt, .control = p.mutation.gate() }; + } + + pub fn prepare(p: *Probe, shape: c.Shape, fill: g.Fill, seed: u64) !bool { + if (shape.rows == 0 or shape.n == 0 or shape.k == 0) return false; + if (p.which == .rows and (shape.k % 8 != 0 or shape.rows > 32)) return false; + if (p.which == .tile and shape.k % 32 != 0) return false; + if (p.d) |d| d.deinit(); + p.d = null; + p.rig.reset(); + const d = try Dense.init(p.a, shape, 1); + p.d = d; + d.fillX(fill, seed, 2); + // W as bf16 bits in [-1, 1), or gate 4's mirrored integers whose halves cancel (partials under 2^24) + const bits = try p.a.alloc(u16, shape.n * shape.k); + defer p.a.free(bits); + var prng = std.Random.DefaultPrng.init(seed ^ 0x6d); + const rng = prng.random(); + const k = shape.k; + for (bits) |*b| b.* = c.toBf16(if (fill == .stress) 0 else rng.float(f32) * 2 - 1); + if (fill == .stress) for (0..shape.n) |j| for (0..k / 2) |i| { + const v: f32 = @floatFromInt(1 + rng.uintLessThan(u32, 7)); + bits[j * k + i] = c.toBf16(v); + bits[j * k + k / 2 + i] = c.toBf16(-v); + }; + for (d.w, bits) |*wv, b| wv.* = c.fromBf16(b); + p.x = try p.rig.input(std.mem.sliceAsBytes(d.x)); + p.w = try p.rig.input(std.mem.sliceAsBytes(bits)); + p.out = try p.rig.output(d.outputs() * c.size(spec(p.which).out)); + return true; + } + + pub fn run(p: *Probe) !void { + const d = p.d.?; + const pass = p.rig.begin(); + const e = pass.e; + e.setPipeline(p.pipe); + e.setBuffer(p.x.buf, p.x.off, 0); + e.setBuffer(p.w.buf, p.w.off, 1); + const args = Args{ .rows = @intCast(d.shape.rows), .n = @intCast(d.shape.n), .k = @intCast(d.shape.k) }; + e.setBytes(std.mem.asBytes(&args), 2); + e.setBuffer(p.out.buf, p.out.off, 3); + if (p.which == .rows) { + e.dispatchGroups(mtl.Size.of((d.shape.n + 7) / 8, (d.shape.rows + 15) / 16, 1), mtl.Size.of(256, 1, 1)); + } else e.dispatchGroups(mtl.Size.of((d.shape.n + 63) / 64, (d.shape.rows + 63) / 64, 1), mtl.Size.of(128, 1, 1)); + try p.rig.finish(pass); + } + + pub fn count(p: *Probe) usize { + return p.d.?.outputs(); + } + + pub fn reference(p: *Probe, want: []f64, mag: []f64) !void { + return p.d.?.reference(want, mag); + } + + pub fn simulate(p: *Probe, acc: c.Accum, order: g.Order, out: []f64) !void { + return p.d.?.simulate(acc, order, out); + } +}; diff --git a/zig/tests/contract/metal_lfm2_confidence.zig b/zig/tests/contract/metal_lfm2_confidence.zig new file mode 100644 index 0000000000..ebf74ea5f2 --- /dev/null +++ b/zig/tests/contract/metal_lfm2_confidence.zig @@ -0,0 +1,110 @@ +//! LFM2's DSpark confidence logit (lfm2_dspark_confidence): a dot of a row and a W1 row with the head's 2,304 weights. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const c = contract.spec; +const g = contract.gates; +const rows = contract.rows; +const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); + +const hidden = 2048; +const rank = 256; +pub const name = "metal lfm2 dspark confidence"; + +/// rows: one stream's drafts (row i reads the block's row i and the W1 row of the token before draft i). +const shapes = [_]c.Shape{ .{ .rows = 1, .n = 1, .k = hidden + rank }, .{ .rows = 4, .n = 1, .k = hidden + rank }, .{ .rows = 9, .n = 1, .k = hidden + rank } }; + +/// A thread's chain over every 256th input, a 32-way simd_sum tree, then the eight simdgroups' partials in order. +fn depth(s: c.Shape) usize { + return (s.k + 255) / 256 + 5 + 8; +} + +pub fn spec() c.Spec { + return .{ .name = name, .accum = .f32, .depth = depth, .out = .f32, .served = &shapes }; +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + pipe: mtl.Pipeline, + mutation: controls.Mutation, + r: ?rows.Rows = null, + args: [6]rig_.Ref = undefined, // x, W1 rows, ids, picks, the head's weights, the shape + out: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, text, "lfm2_dspark_confidence"), .mutation = mutation }; + } + + pub fn deinit(p: *Probe) void { + if (p.r) |r| r.deinit(); + p.rig.deinit(); + p.pipe.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + var s = spec(); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() }; + } + + /// Row i's first 2,048 inputs as the block's row i, its last 256 as W1 row i: the token before draft i is i. + pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { + if (s.k != hidden + rank or s.rows == 0 or s.rows > 16) return false; + if (p.r) |r| r.deinit(); + p.r = null; + p.rig.reset(); + const r = try rows.Rows.init(p.a, .dot, s); + p.r = r; + r.fill(f, seed); + const x = try p.a.alloc(u16, s.rows * hidden); + defer p.a.free(x); + const w1 = try p.a.alloc(u16, s.rows * rank); + defer p.a.free(w1); + for (0..s.rows) |i| { + @memcpy(x[i * hidden ..][0..hidden], r.x[i * s.k ..][0..hidden]); + @memcpy(w1[i * rank ..][0..rank], r.x[i * s.k + hidden ..][0..rank]); + } + // the block's pending id is token 0; the pick before draft i (i > 0) is token i, a flag word after each + var picks: [32]u32 = @splat(0xdead); + for (1..s.rows) |i| picks[2 * (i - 1)] = @intCast(i); + const ids = [1]u32{0}; + const args = [5]u32{ @intCast(s.rows), 1, @intCast(s.rows), rank, hidden }; + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(x)); + p.args[1] = try p.rig.input(std.mem.sliceAsBytes(w1)); + p.args[2] = try p.rig.input(std.mem.asBytes(&ids)); + p.args[3] = try p.rig.input(std.mem.asBytes(&picks)); + p.args[4] = try p.rig.input(std.mem.sliceAsBytes(r.w)); + p.args[5] = try p.rig.input(std.mem.asBytes(&args)); + p.out = try p.rig.output(s.rows * 4); + return true; + } + + pub fn run(p: *Probe) !void { + const n = p.r.?.shape.rows; + const pass = p.rig.begin(); + const e = pass.e; + e.setPipeline(p.pipe); + for (p.args, 0..) |ref, i| e.setBuffer(ref.buf, ref.off, i); + e.setBuffer(p.out.buf, p.out.off, 6); + e.dispatchGroups(mtl.Size.of(n, 1, 1), mtl.Size.of(256, 1, 1)); + try p.rig.finish(pass); + } + + pub fn count(p: *Probe) usize { + return p.r.?.outputs(); + } + + pub fn reference(p: *Probe, want: []f64, mag: []f64) !void { + return p.r.?.reference(want, mag); + } + + pub fn simulate(p: *Probe, acc: c.Accum, order: g.Order, out: []f64) !void { + return p.r.?.simulate(acc, order, out); + } +}; diff --git a/zig/tests/contract/metal_lfm2_controls.zig b/zig/tests/contract/metal_lfm2_controls.zig index 655a97d4fc..bd8a890529 100644 --- a/zig/tests/contract/metal_lfm2_controls.zig +++ b/zig/tests/contract/metal_lfm2_controls.zig @@ -9,6 +9,19 @@ pub const Mutation = enum { argmax_no_flag, conv_first_state, attention_first_cache, + tap_packed_rows, + tap_clobbers_column, + markov_nan_dropped, + markov_first_prev, + draft_ids_unzeroed, + block_attention_causal, + confidence_no_w1, + markov_bf16_nan_dropped, + markov_bf16_first_prev, + bf16_rows_short_k, + bf16_rows_last_row, + bf16_tile_untransposed, + bf16_tile_last_column, pub fn name(m: Mutation) []const u8 { return switch (m) { @@ -17,6 +30,19 @@ pub const Mutation = enum { .argmax_no_flag => "control: metal lfm2 argmax, a nonfinite logit is not flagged", .conv_first_state => "control: metal lfm2 stream conv, every stream reads the first stream's state", .attention_first_cache => "control: metal lfm2 stream attention, every stream writes the first stream's cache", + .tap_packed_rows => "control: metal lfm2 tap copy, rows packed at the hidden size, not the taps' row", + .tap_clobbers_column => "control: metal lfm2 tap copy, a column of another tap is overwritten", + .markov_nan_dropped => "control: metal lfm2 markov step, a NaN score is not flagged", + .markov_first_prev => "control: metal lfm2 markov step, every stream reads the first stream's previous token", + .draft_ids_unzeroed => "control: metal lfm2 draft ids, a pick past the vocabulary keeps its token", + .confidence_no_w1 => "control: metal lfm2 dspark confidence, the previous token's W1 row is not added", + .block_attention_causal => "control: metal lfm2 block attention, a row misses the last key", + .markov_bf16_nan_dropped => "control: metal lfm2 markov step bf16, a NaN score is not flagged", + .markov_bf16_first_prev => "control: metal lfm2 markov step bf16, every stream reads the first stream's previous token", + .bf16_rows_short_k => "control: metal lfm2 bf16 rows, a lane skips every second step of k", + .bf16_rows_last_row => "control: metal lfm2 bf16 rows, a group's last row is not computed", + .bf16_tile_untransposed => "control: metal lfm2 bf16 tile, W's tile is read untransposed", + .bf16_tile_last_column => "control: metal lfm2 bf16 tile, the last column is not written", }; } @@ -24,7 +50,7 @@ pub const Mutation = enum { pub fn gate(m: Mutation) ?contract.gates.Gate { return switch (m) { .none => null, - .argmax_no_flag => .nonfinite, + .argmax_no_flag, .markov_nan_dropped, .markov_bf16_nan_dropped => .nonfinite, else => .shapes, }; } @@ -38,6 +64,19 @@ pub const Mutation = enum { .argmax_no_flag => .{ .from = "if (!isfinite(f)) bad = 1;", .to = "if (false) bad = 1;" }, .conv_first_state => .{ .from = "lfm2_conv_row(bcx + size_t(s.base) * 3 * D, s.conv,", .to = "lfm2_conv_row(bcx + size_t(s.base) * 3 * D, segs[0].conv," }, .attention_first_cache => .{ .from = "queries + b * QH * HD, s.keys, s.values,", .to = "queries + b * QH * HD, segs[0].keys, segs[0].values," }, + .tap_packed_rows => .{ .from = "(taps + size_t(i.y) * at.x + at.y))[i.x]", .to = "(taps + size_t(i.y) * D + at.y))[i.x]" }, + .markov_nan_dropped => .{ .from = "nan = nan || isnan(sc);", .to = "nan = false;" }, + .markov_first_prev => .{ .from = "id = prev[size_t(s) * p.prev_stride];", .to = "id = prev[0];" }, + .draft_ids_unzeroed => .{ .from = "ids[i] = ok ? token : 0u;", .to = "ids[i] = token;" }, + .confidence_no_w1 => .{ .from = "if (id < p.vocab)\n for (uint k = t; k < p.rank", .to = "if (false)\n for (uint k = t; k < p.rank" }, + .block_attention_causal => .{ .from = "count[b] = FULL ? p.len + 1 :", .to = "count[b] = FULL ? p.len :" }, + .markov_bf16_nan_dropped => .{ .from = "nan = nan || isnan(sc);", .to = "nan = false;" }, + .markov_bf16_first_prev => .{ .from = "id = prev[size_t(s) * p.prev_stride];", .to = "id = prev[0];" }, + .bf16_rows_short_k => .{ .from = "for (uint i = 8 * lane; i < p.k; i += 256) {", .to = "for (uint i = 8 * lane; i < p.k; i += 512) {" }, + .bf16_rows_last_row => .{ .from = "if (r >= rows) break;", .to = "if (r + 1 >= rows) break;" }, + .bf16_tile_untransposed => .{ .from = "32, ulong2(0, 0), true);", .to = "32, ulong2(0, 0), false);" }, + .bf16_tile_last_column => .{ .from = "if (row < p.rows && col < p.n) y[", .to = "if (row < p.rows && col + 1 < p.n) y[" }, + .tap_clobbers_column => .{ .from = "[i.x] = from[i.x];\n}\n", .to = "[i.x] = from[i.x];\n *(device bfloat*)(taps + size_t(i.y) * at.x) = 0;\n}\n" }, }; } }; diff --git a/zig/tests/contract/metal_lfm2_dspark.zig b/zig/tests/contract/metal_lfm2_dspark.zig new file mode 100644 index 0000000000..04b7fc489a --- /dev/null +++ b/zig/tests/contract/metal_lfm2_dspark.zig @@ -0,0 +1,264 @@ +//! LFM2's DSpark kernels: the Markov step (scan then pick), and held drafts into a verify's ids. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const c = contract.spec; +const g = contract.gates; +const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); + +pub const Which = enum { markov_step, draft_ids, markov_step_bf16 }; +const rank = 256; +const vocab = 128000; +const span = 512; // tokens a scan threadgroup takes (the kernel's MARKOV_SPAN) +const groups = (vocab + span - 1) / span; +const table = 1000; // W1's rows in the check: ids are inside it or past the vocabulary (a zero row) +const names = [_][]const u8{ "metal lfm2 markov step", "metal lfm2 draft ids", "metal lfm2 markov step bf16" }; +const stride = 64; // words between a stream's picks: a hold pass's 32 blocks, a token and a flag each + +const ids_shapes = [_]c.Shape{ .{ .rows = 1, .n = 1, .k = 1 }, .{ .rows = 4, .n = 1, .k = 1 }, .{ .rows = 15, .n = 1, .k = 1 } }; +const step_shapes = [_]c.Shape{ .{ .rows = 1, .n = vocab, .k = rank }, .{ .rows = 4, .n = vocab, .k = rank }, .{ .rows = 9, .n = vocab, .k = rank } }; + +fn none(_: c.Shape) usize { + return 0; +} + +fn depth(_: c.Shape) usize { + return rank; +} + +pub fn spec(w: Which) c.Spec { + return switch (w) { + .draft_ids => .{ .name = names[1], .accum = .none, .depth = none, .out = .raw, .served = &ids_shapes, .sums = .{ .f16 = 0, .bf16 = 0, .why = "a copy moves tokens: no sums" } }, + .markov_step, .markov_step_bf16 => .{ .name = if (w == .markov_step) names[0] else names[2], .accum = .none, .depth = depth, .out = .raw, .served = &step_shapes, .nan = .{ .stated = "a row with a NaN score is flagged nonfinite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "picks are tokens: the check bounds each pick's score by gamma(256), not sum bits" } }, + }; +} + +/// A bf16 value for element `i` over signs and binades; the nonfinite fill plants NaN and infinities. +fn value(i: usize, seed: u64, fill: g.Fill) u16 { + if (fill == .nonfinite) switch ((@as(u64, i) *% 2654435761 +% seed) % 9973) { + 0 => return 0x7fc0, + 1 => return 0x7f80, + 2 => return 0xff80, + else => {}, + }; + const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40; + const mant: f32 = @as(f32, @floatFromInt(h % 1024)) / 1024 + 1; + const exp: i32 = @as(i32, @intCast((h >> 10) % 8)) - 4; + const v = std.math.ldexp(mant, exp) * @as(f32, if (((h >> 15) & 1) == 0) 1 else -1); + return c.toBf16(v); +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + which: Which, + mutation: controls.Mutation, + pipes: [2]mtl.Pipeline, + rows: usize = 0, + src: []u16 = &.{}, // the base logits rows + w1: []u16 = &.{}, + codes: []u8 = &.{}, + scales: []u16 = &.{}, + biases: []u16 = &.{}, + w2: []u16 = &.{}, // the bf16 step's W2 + ids: []u32 = &.{}, + args: [8]rig_.Ref = undefined, + y: rig_.Ref = undefined, + bad: rig_.Ref = undefined, // draft_ids: the flag word it sets for a flagged or invalid pick, zeroed each run + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + const kernels: [2][:0]const u8 = switch (w) { + .markov_step => .{ "lfm2_markov_scan", "lfm2_markov_pick" }, + .markov_step_bf16 => .{ "lfm2_markov_scan_bf16", "lfm2_markov_pick" }, + .draft_ids => .{ "lfm2_draft_ids", "lfm2_draft_ids" }, + }; + const first = try rig_.pipeline(device, text, kernels[0]); + errdefer first.deinit(); + const second = try rig_.pipeline(device, text, kernels[1]); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .which = w, .mutation = mutation, .pipes = .{ first, second } }; + } + + fn drop(p: *Probe) void { + inline for (.{ "src", "w1", "codes", "scales", "biases", "w2", "ids" }) |f| { + p.a.free(@field(p, f)); + @field(p, f) = &.{}; + } + p.rig.reset(); + } + + pub fn deinit(p: *Probe) void { + p.drop(); + p.rig.deinit(); + for (p.pipes) |x| x.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + var s = spec(p.which); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() }; + } + + fn fill(out: []u16, seed: u64, f: g.Fill) void { + for (out, 0..) |*v, i| v.* = value(i, seed, f); + } + + pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { + p.drop(); + p.rows = s.rows; + if (p.which == .draft_ids) { // picks a stride apart: tokens inside the vocabulary, its last, and past it + if (s.n != 1 or s.rows == 0 or s.rows > 16) return false; + p.ids = try p.a.alloc(u32, s.rows * stride); + for (p.ids, 0..) |*id, i| id.* = switch ((i / stride) % 3) { + 0 => @intCast((@as(u64, i) * 7919 + seed) % vocab), + 1 => vocab - 1, + else => vocab + @as(u32, @intCast(i % 5)), + }; + // the flag word after each pick: zero, but planted nonzero on the last of every four streams + for (0..s.rows) |r| p.ids[r * stride + 1] = if (r % 4 == 3) 1 else 0; + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.ids)); + const at = [4]u32{ stride, @intCast(s.rows), vocab, 0 }; + p.args[1] = try p.rig.input(std.mem.asBytes(&at)); + p.y = try p.rig.output(p.count()); + return true; + } + if (s.n != vocab or s.k != rank or s.rows == 0) return false; + // each stream's step row sits a block (two rows here) apart, as a hold's rows of one step do + p.src = try p.a.alloc(u16, 2 * s.rows * vocab); + fill(p.src, seed, f); + p.w1 = try p.a.alloc(u16, table * rank); + fill(p.w1, seed +% 5, .random); + p.codes = try p.a.alloc(u8, vocab * rank); + for (p.codes, 0..) |*q, i| q.* = @truncate((@as(u64, i) *% 2654435761 +% seed) >> 11); + p.scales = try p.a.alloc(u16, vocab * rank / 64); + p.biases = try p.a.alloc(u16, vocab * rank / 64); + for (p.scales, p.biases, 0..) |*sc, *bi, i| { + sc.* = c.toBf16(@as(f32, @floatFromInt(i % 7 + 1)) / 4096); + bi.* = c.toBf16(@as(f32, @floatFromInt(i % 5)) / 64 - 0.03); + } + if (p.which == .markov_step_bf16) { + p.w2 = try p.a.alloc(u16, vocab * rank); + fill(p.w2, seed +% 9, .random); + } + // the previous token every second word (the picks' layout): inside W1, its last row, or past the vocabulary + p.ids = try p.a.alloc(u32, 2 * s.rows); + for (p.ids, 0..) |*id, i| id.* = if (i % 2 == 1) 0xdead else switch ((i / 2) % 3) { + 0 => @intCast((@as(u64, i) * 37 + seed) % table), + 1 => table - 1, + else => vocab + @as(u32, @intCast(i)), + }; + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.src)); + p.args[1] = try p.rig.input(std.mem.sliceAsBytes(p.w1)); + p.args[2] = if (p.which == .markov_step_bf16) try p.rig.input(std.mem.sliceAsBytes(p.w2)) else try p.rig.input(p.codes); + p.args[3] = try p.rig.input(std.mem.sliceAsBytes(p.ids)); + const args = [4]u32{ 2 * vocab, rank, vocab, 2 }; + p.args[4] = try p.rig.input(std.mem.asBytes(&args)); + p.args[5] = try p.rig.input(std.mem.sliceAsBytes(p.scales)); + p.args[6] = try p.rig.input(std.mem.sliceAsBytes(p.biases)); + const n: u32 = groups; + p.args[7] = try p.rig.input(std.mem.asBytes(&n)); + p.y = try p.rig.output(p.count()); + return true; + } + + pub fn count(p: *Probe) usize { + return switch (p.which) { + .markov_step, .markov_step_bf16 => p.rows * 8, + .draft_ids => p.rows * 4, + }; + } + + pub fn run(p: *Probe) !void { + const pass = p.rig.begin(); + const e = pass.e; + if (p.which == .draft_ids) { + e.setPipeline(p.pipes[0]); + e.setBuffer(p.args[0].buf, p.args[0].off, 0); + e.setBuffer(p.args[1].buf, p.args[1].off, 1); + e.setBuffer(p.y.buf, p.y.off, 2); + p.bad = try p.rig.scratch(4); + @memset(p.rig.bytes(p.bad, 4), 0); + e.setBuffer(p.bad.buf, p.bad.off, 3); + e.dispatchThreads(mtl.Size.of(p.rows, 1, 1), mtl.Size.of(32, 1, 1)); + return p.rig.finish(pass); + } + const best = try p.rig.scratch(p.rows * groups * 4); + const at = try p.rig.scratch(p.rows * groups * 4); + e.setPipeline(p.pipes[0]); + for (p.args[0..5], 0..) |r, i| e.setBuffer(r.buf, r.off, i); + e.setBuffer(best.buf, best.off, 5); + e.setBuffer(at.buf, at.off, 6); + if (p.which == .markov_step) { + e.setBuffer(p.args[5].buf, p.args[5].off, 7); + e.setBuffer(p.args[6].buf, p.args[6].off, 8); + } + e.dispatchGroups(mtl.Size.of(groups, p.rows, 1), mtl.Size.of(256, 1, 1)); + e.setPipeline(p.pipes[1]); + e.setBuffer(best.buf, best.off, 0); + e.setBuffer(at.buf, at.off, 1); + e.setBuffer(p.args[7].buf, p.args[7].off, 2); + e.setBuffer(p.y.buf, p.y.off, 3); + e.dispatchGroups(mtl.Size.of(p.rows, 1, 1), mtl.Size.of(256, 1, 1)); + try p.rig.finish(pass); + } + + pub fn reference(_: *Probe, want: []f64, mag: []f64) !void { + @memset(mag, 0); + @memset(want, 0); // the step and the ids check themselves (check) + } + + /// Each stream's pick against float64 scores with the kernel's bf16 roundings: the best, but for near ties. + pub fn check(p: *Probe, _: g.Gate) !?[]const u8 { + const out = p.rig.bytes(p.y, p.count()); + if (p.which == .draft_ids) { // exact: each stream's first pick, or 0 for a token past the vocabulary or a flag + var any = false; + for (0..p.rows) |r| { + const token = p.ids[r * stride]; + const ok = token < vocab and p.ids[r * stride + 1] == 0; + any = any or !ok; + if (std.mem.readInt(u32, out[4 * r ..][0..4], .little) != (if (ok) token else 0)) return "an id differs from its pick"; + } + const raised = std.mem.readInt(u32, p.rig.bytes(p.bad, 4)[0..4], .little); + if ((raised != 0) != any) return "the bad flag does not follow the flagged and invalid picks"; + return null; + } + for (0..p.rows) |s| { + const token = std.mem.readInt(u32, out[8 * s ..][0..4], .little); + const flag = std.mem.readInt(u32, out[8 * s + 4 ..][0..4], .little); + const id = p.ids[2 * s]; + var top: f64 = -std.math.inf(f64); + var mine: ?f64 = null; + var nan = false; + var bound: f64 = 0; + for (0..vocab) |v| { + var dot: f64 = 0; + var size: f64 = 0; + if (id < table) for (0..rank) |i| { + const k = (v * rank + i) / 64; + const w: f64 = if (p.which == .markov_step_bf16) c.fromBf16(p.w2[v * rank + i]) else @as(f64, c.fromBf16(p.scales[k])) * @as(f64, @floatFromInt(p.codes[v * rank + i])) + c.fromBf16(p.biases[k]); + dot += w * c.fromBf16(p.w1[id * rank + i]); + size += @abs(w * c.fromBf16(p.w1[id * rank + i])); + }; + const base: f32 = c.fromBf16(p.src[2 * s * vocab + v]); + const score: f64 = c.fromBf16(c.toBf16(base + c.fromBf16(c.toBf16(@floatCast(dot))))); + if (std.math.isNan(score)) nan = true; + if (score > top) top = score; + if (v == token) mine = score; + // two bf16 steps of the score and the fp32 sum's own error + bound = @max(bound, @abs(score) * 0x1p-7 + c.gamma(rank, 0x1p-24) * size); + } + if (nan) { + if (flag != 1) return "a row with a NaN score is not flagged"; + continue; + } + if (flag != 0) return "a finite row is flagged nonfinite"; + const got = mine orelse return "the pick is past the vocabulary"; + if (got < top - bound) return "the pick's score is below the row's best by more than its bound"; + } + return null; + } +}; diff --git a/zig/tests/contract/metal_lfm2_tap.zig b/zig/tests/contract/metal_lfm2_tap.zig new file mode 100644 index 0000000000..021baffc9e --- /dev/null +++ b/zig/tests/contract/metal_lfm2_tap.zig @@ -0,0 +1,137 @@ +//! LFM2's tap copy for an external drafter (lfm2_tap): a layer's rows into their tap's columns, the others kept. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const c = contract.spec; +const g = contract.gates; +const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); + +const hidden = 2048; +const taps = 5; +pub const name = "metal lfm2 tap copy"; + +/// rows: one decode row, a verify window, a shared round's 32 and a 2,048-row prompt chunk. +const shapes = [_]c.Shape{ .{ .rows = 1, .n = taps * hidden, .k = 1 }, .{ .rows = 10, .n = taps * hidden, .k = 1 }, .{ .rows = 32, .n = taps * hidden, .k = 1 }, .{ .rows = 2048, .n = taps * hidden, .k = 1 } }; + +fn none(_: c.Shape) usize { + return 0; +} + +pub fn spec() c.Spec { + return .{ .name = name, .accum = .none, .depth = none, .out = .bf16, .served = &shapes, .sums = .{ .f16 = 0, .bf16 = 0, .why = "a copy moves values: no sums" } }; +} + +/// A bf16 value for element `i` over signs and binades; the nonfinite fill plants NaN and infinities. +fn value(i: usize, seed: u64, fill: g.Fill) u16 { + if (fill == .nonfinite) switch ((@as(u64, i) *% 2654435761 +% seed) % 9973) { + 0 => return 0x7fc0, + 1 => return 0x7f80, + 2 => return 0xff80, + else => {}, + }; + const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40; + const mant: f32 = @as(f32, @floatFromInt(h % 1024)) / 1024 + 1; + const exp: i32 = @as(i32, @intCast((h >> 10) % 8)) - 4; + const v = std.math.ldexp(mant, exp) * @as(f32, if (((h >> 15) & 1) == 0) 1 else -1); + return c.toBf16(v); +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + pipe: mtl.Pipeline, + mutation: controls.Mutation, + rows: usize = 0, + src: []u16 = &.{}, // H rows + prior: []u16 = &.{}, // the taps rows as the call finds them (the other taps' columns must stay) + args: [2]rig_.Ref = undefined, + y: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, text, "lfm2_tap"), .mutation = mutation }; + } + + fn drop(p: *Probe) void { + p.a.free(p.src); + p.a.free(p.prior); + p.src = &.{}; + p.prior = &.{}; + p.rig.reset(); + } + + pub fn deinit(p: *Probe) void { + p.drop(); + p.rig.deinit(); + p.pipe.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + var s = spec(); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() }; + } + + fn fill(out: []u16, seed: u64, f: g.Fill) void { + for (out, 0..) |*v, i| v.* = value(i, seed, f); + } + + pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { + p.drop(); + if (s.n != taps * hidden or s.rows == 0) return false; + p.rows = s.rows; + p.src = try p.a.alloc(u16, s.rows * hidden); + fill(p.src, seed, f); + p.prior = try p.a.alloc(u16, s.rows * taps * hidden); + fill(p.prior, seed +% 3, .random); + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.src)); + const at = [2]u32{ taps * hidden, 2 * hidden }; // the third tap's columns + p.args[1] = try p.rig.input(std.mem.asBytes(&at)); + p.y = try p.rig.output(p.count() * 2); + return true; + } + + pub fn count(p: *Probe) usize { + return p.rows * taps * hidden; + } + + pub fn run(p: *Probe) !void { + @memcpy(p.rig.bytes(p.y, p.count() * 2), std.mem.sliceAsBytes(p.prior)); + const pass = p.rig.begin(); + const e = pass.e; + e.setPipeline(p.pipe); + e.setBuffer(p.args[0].buf, p.args[0].off, 0); + e.setBuffer(p.y.buf, p.y.off, 1); + e.setBuffer(p.args[1].buf, p.args[1].off, 2); + e.dispatchThreads(mtl.Size.of(hidden / 8, p.rows, 1), mtl.Size.of(256, 1, 1)); + return p.rig.finish(pass); + } + + pub fn reference(p: *Probe, want: []f64, mag: []f64) !void { + for (want, mag, p.prior) |*w, *m, v| { // the other taps' columns keep what the call found + w.* = c.fromBf16(v); + m.* = @abs(w.*); + } + for (0..p.rows) |r| for (0..hidden) |i| { + const v = c.fromBf16(p.src[r * hidden + i]); + want[r * taps * hidden + 2 * hidden + i] = v; + mag[r * taps * hidden + 2 * hidden + i] = @abs(v); + }; + } + + /// The copy is exact: the third tap's columns hold H's bits, the others what the call found. + pub fn check(p: *Probe, _: g.Gate) !?[]const u8 { + const out = std.mem.bytesAsSlice(u16, p.rig.bytes(p.y, p.count() * 2)); + for (out, p.prior, 0..) |got, before, i| { + const r = i / (taps * hidden); + const col = i % (taps * hidden); + const want = if (col >= 2 * hidden and col < 3 * hidden) p.src[r * hidden + col - 2 * hidden] else before; + if (got != want) return "an output differs from the copy's bits"; + } + return null; + } +}; diff --git a/zig/tests/contract/metal_main.zig b/zig/tests/contract/metal_main.zig index 46801e171c..52a0d205c3 100644 --- a/zig/tests/contract/metal_main.zig +++ b/zig/tests/contract/metal_main.zig @@ -27,6 +27,11 @@ const gemma_mixed = @import("metal_gemma4_mixed.zig"); const lfm2_argmax = @import("metal_lfm2_argmax.zig"); const lfm2_streams = @import("metal_lfm2_streams.zig"); const lfm2_controls = @import("metal_lfm2_controls.zig"); +const lfm2_tap = @import("metal_lfm2_tap.zig"); +const lfm2_dspark = @import("metal_lfm2_dspark.zig"); +const lfm2_bf16 = @import("metal_lfm2_bf16.zig"); +const lfm2_attn = @import("metal_lfm2_attn.zig"); +const lfm2_conf = @import("metal_lfm2_confidence.zig"); const g = contract.gates; const usage = "usage: tf-kernel-contract [--only TEXT] [--no-gates] [--no-limits] [--offline DIR] [--seed N] [--repro] [--rotary-long] [--rotary-accuracy FILE]"; @@ -191,6 +196,101 @@ pub fn main(init: std.process.Init) !u8 { try kernels.append(a, p.kernel()); } } + var tap_list: std.ArrayList(*lfm2_tap.Probe) = .empty; + defer { + for (tap_list.items) |p| { + p.deinit(); + a.destroy(p); + } + tap_list.deinit(a); + } + for ([_]lfm2_controls.Mutation{ .none, .tap_packed_rows, .tap_clobbers_column }) |m| { + if (!wanted(o, if (m == .none) lfm2_tap.name else m.name())) continue; + const p = try a.create(lfm2_tap.Probe); + errdefer a.destroy(p); + p.* = try lfm2_tap.Probe.init(a, device, queue, m); + try tap_list.append(a, p); + try kernels.append(a, p.kernel()); + } + var drafter_list: std.ArrayList(*lfm2_dspark.Probe) = .empty; + var block_list: std.ArrayList(*lfm2_attn.Probe) = .empty; + var bf16_list: std.ArrayList(*lfm2_bf16.Probe) = .empty; + defer { + for (drafter_list.items) |p| { + p.deinit(); + a.destroy(p); + } + for (block_list.items) |p| { + p.deinit(); + a.destroy(p); + } + for (bf16_list.items) |p| { + p.deinit(); + a.destroy(p); + } + drafter_list.deinit(a); + block_list.deinit(a); + bf16_list.deinit(a); + } + for (std.enums.values(lfm2_controls.Mutation)) |m| { + const drafter_which: ?lfm2_dspark.Which = switch (m) { + .markov_nan_dropped, .markov_first_prev => .markov_step, + .markov_bf16_nan_dropped, .markov_bf16_first_prev => .markov_step_bf16, + .draft_ids_unzeroed => .draft_ids, + else => null, + }; + if (m == .none or drafter_which != null) { + for (std.enums.values(lfm2_dspark.Which)) |w| { + if (m != .none and w != drafter_which.?) continue; + if (!wanted(o, if (m == .none) lfm2_dspark.spec(w).name else m.name())) continue; + const p = try a.create(lfm2_dspark.Probe); + errdefer a.destroy(p); + p.* = try lfm2_dspark.Probe.init(a, device, queue, w, m); + try drafter_list.append(a, p); + try kernels.append(a, p.kernel()); + } + } + const bf16_which: ?lfm2_bf16.Which = switch (m) { + .bf16_rows_short_k, .bf16_rows_last_row => .rows, + .bf16_tile_untransposed, .bf16_tile_last_column => .tile, + else => null, + }; + if (m == .none or bf16_which != null) { + for (std.enums.values(lfm2_bf16.Which)) |w| { + if (m != .none and w != bf16_which.?) continue; + if (!wanted(o, if (m == .none) lfm2_bf16.spec(w).name else m.name())) continue; + const p = try a.create(lfm2_bf16.Probe); + errdefer a.destroy(p); + p.* = try lfm2_bf16.Probe.init(a, device, queue, w, m); + try bf16_list.append(a, p); + try kernels.append(a, p.kernel()); + } + } + if (m == .none or m == .block_attention_causal) { + if (!wanted(o, if (m == .none) lfm2_attn.name else m.name())) continue; + const p = try a.create(lfm2_attn.Probe); + errdefer a.destroy(p); + p.* = try lfm2_attn.Probe.init(a, device, queue, m); + try block_list.append(a, p); + try kernels.append(a, p.kernel()); + } + } + var conf_list: std.ArrayList(*lfm2_conf.Probe) = .empty; + defer { + for (conf_list.items) |p| { + p.deinit(); + a.destroy(p); + } + conf_list.deinit(a); + } + for ([_]lfm2_controls.Mutation{ .none, .confidence_no_w1 }) |m| { + if (!wanted(o, if (m == .none) lfm2_conf.name else m.name())) continue; + const p = try a.create(lfm2_conf.Probe); + errdefer a.destroy(p); + p.* = try lfm2_conf.Probe.init(a, device, queue, m); + try conf_list.append(a, p); + try kernels.append(a, p.kernel()); + } var rows_list: std.ArrayList(*rows_k.Row) = .empty; var proj_list: std.ArrayList(*affine_k.Proj) = .empty; defer for (rows_list.items) |r| r.deinit(); diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig new file mode 100644 index 0000000000..4fb630fb13 --- /dev/null +++ b/zig/tests/lfm2_dspark.zig @@ -0,0 +1,398 @@ +//! LFM2's DSpark drafter on the GPU against an fp32 host reference of the released model, on the real weights. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const q = tf.lfm2_moe; +const c = q.config; +const st = q.state; +const ck = tf.checkpoint; +const quant = q.dspark_quant; + +const hd = c.head_dim; +const kw = c.kv_heads * hd; +const qw = c.query_heads * hd; + +/// The drafter's tensors as fp32 on the host, in the checkpoint's own layout (interleaved RoPE pairs). +const Ref = struct { + gpa: std.mem.Allocator, + check: ck.Checkpoint, + cfg: q.dspark_config.Config, + + fn get(r: *const Ref, name: []const u8) ![]f32 { + const t = try r.check.get(name); + const src = t.host(u16); + const out = try r.gpa.alloc(f32, src.len); + for (out, src) |*o, v| o.* = quant.toF32(v); + return out; + } + + fn layer(r: *const Ref, buf: []u8, i: usize, part: []const u8) ![]f32 { + return r.get(try std.fmt.bufPrint(buf, "layers.{d}.{s}.weight", .{ i, part })); + } +}; + +/// y[n] = W[n, k] x[k] for each of `rows` rows, on every core. +fn matmul(gpa: std.mem.Allocator, w: []const f32, x: []const f32, y: []f32, n: usize, k: usize, rows: usize) !void { + const Job = struct { + fn run(w_: []const f32, x_: []const f32, y_: []f32, n_: usize, k_: usize, rows_: usize, from: usize, to: usize) void { + for (0..rows_) |r| for (from..to) |col| { + var s: f32 = 0; + const wr = w_[col * k_ ..][0..k_]; + const xr = x_[r * k_ ..][0..k_]; + for (wr, xr) |a, b| s += a * b; + y_[r * n_ + col] = s; + }; + } + }; + const threads = @min(std.Thread.getCpuCount() catch 1, 16); + const per = (n + threads - 1) / threads; + const hs = try gpa.alloc(std.Thread, threads); + defer gpa.free(hs); + var made: usize = 0; + defer for (hs[0..made]) |h| h.join(); // a failed spawn still joins the workers already running + var at: usize = 0; + while (at < n) : (at += per) { + hs[made] = try std.Thread.spawn(.{}, Job.run, .{ w, x, y, n, k, rows, at, @min(n, at + per) }); + made += 1; + } +} + +fn rms(x: []f32, gain: []const f32) void { + var ss: f32 = 0; + for (x) |v| ss += v * v; + const inv = 1 / @sqrt(ss / @as(f32, @floatFromInt(x.len)) + c.eps); + for (x, gain) |*v, g| v.* = v.* * inv * g; +} + +/// Interleaved RoPE at `pos` over each head of `x` (pairs 2i, 2i + 1). +fn rope(x: []f32, pos: u64, theta: f32) void { + var h: usize = 0; + while (h < x.len) : (h += hd) for (0..hd / 2) |i| { + const angle = @as(f32, @floatFromInt(pos)) * std.math.pow(f32, theta, -@as(f32, @floatFromInt(i)) / (hd / 2)); + const a = x[h + 2 * i]; + const b = x[h + 2 * i + 1]; + x[h + 2 * i] = a * @cos(angle) - b * @sin(angle); + x[h + 2 * i + 1] = a * @sin(angle) + b * @cos(angle); + }; +} + +/// The target's embedding row `id`, dequantized from its codes at the checkpoint's own width (scale * q + bias). +fn embedRow(m: *q.Model, id: u32, out: []f32) void { + const e = m.weights.embedding; + const bits: usize = e.bits; + const row_bytes = c.hidden * bits / 8; + const codes = (e.w.contents() + e.w_off)[@as(usize, id) * row_bytes ..][0..row_bytes]; + const sc = @as([*]const u16, @ptrCast(@alignCast(e.scales.contents() + e.s_off)))[@as(usize, id) * c.hidden / 64 ..][0 .. c.hidden / 64]; + const bi = @as([*]const u16, @ptrCast(@alignCast(e.biases.contents() + e.b_off)))[@as(usize, id) * c.hidden / 64 ..][0 .. c.hidden / 64]; + for (0..c.hidden) |j| { + // codes are a little-endian bit stream: code j takes bits [j * bits, (j + 1) * bits) + const at = j * bits; + var word: u32 = codes[at / 8]; + if (at / 8 + 1 < row_bytes) word |= @as(u32, codes[at / 8 + 1]) << 8; + const qv = (word >> @intCast(at % 8)) & ((@as(u32, 1) << @intCast(bits)) - 1); + out[j] = quant.toF32(sc[j / 64]) * @as(f32, @floatFromInt(qv)) + quant.toF32(bi[j / 64]); + } +} + +/// GPU bf16 row against the reference's row + bias: cosine and worst difference. +fn compare(label: []const u8, gpu: []const u16, row: []const f32, bias: []const f32) void { + var dot: f64 = 0; + var gg: f64 = 0; + var rr: f64 = 0; + var worst: f32 = 0; + for (gpu, row, bias) |g, b, m2| { + const x = quant.toF32(g); + const y = b + m2; + dot += x * y; + gg += x * x; + rr += y * y; + worst = @max(worst, @abs(x - y)); + } + std.debug.print("{s}: cosine {d:.5}, worst difference {d:.3}\n", .{ label, dot / @sqrt(gg * rr), worst }); +} + +/// A hold's GPU ms: the blocks held in a signalled command buffer, waited on and read for its GPU times. +fn gpuHold(d: *q.dspark.Drafter, blocks: []const q.dspark_block.Block) !f64 { + d.signalled += 1; + const cb = (try q.dspark_block.hold(d, blocks, .{ .event = d.event, .value = d.signalled })).?; + defer cb.release(); + cb.wait(); + if (cb.failure() != null) return error.GpuFailed; + return (cb.gpuEnd() - cb.gpuStart()) * 1000; +} + +fn absorb(d: *q.dspark.Drafter, piece: q.dspark_block.Piece) !void { + const run = q.dspark_block.begin(d); + try q.dspark_block.absorbPass(d, run.e, &.{piece}); + try q.dspark_block.end(run.cb, run.e); +} + +/// Served rounds through the drafter's interface: after four warm rounds, no host allocation and one hold pass a round. +fn steady(gpa: std.mem.Allocator, d: *q.dspark.Drafter, taps: mtl.Buffer, ids: []const u32, ctx: usize) !void { + const lanes = tf.lanes; + var s = try lanes.Stream.init(gpa, .{ .id = "steady", .prompt = ids[0 .. ctx + 1], .max_new = 160 }); + defer s.deinit(gpa); + const dr = d.drafter(); + try dr.open(&s); + defer dr.release(&s); + const width: u32 = @intCast(d.tapBytes()); + const feats = struct { + fn at(buf: mtl.Buffer, row: usize, rows: usize, w: u32) lanes.backend.Features { + return .{ .buffer = @intFromPtr(buf.id), .offset = row * w, .rows = @intCast(rows), .row_bytes = w, .space = .device, .dtype = .bf16 }; + } + }; + try dr.absorb(&.{.{ .stream = &s, .features = feats.at(taps, 0, ctx, width), .start = 0, .follow = &.{} }}); + var counted = std.testing.FailingAllocator.init(gpa, .{}); + var anchor: u64 = ctx; + const rounds = 24; + const warm = 4; + var holds: u64 = 0; + for (0..rounds) |r| { + if (r == warm) { + d.gpa = counted.allocator(); + holds = d.signalled; + } + try dr.hold(&.{.{ .stream = &s, .pending = ids[@intCast(anchor % ctx)], .position = anchor + 1, .depth = 3 }}); + var chances: [3]f64 = undefined; + _ = try dr.chances(&s, &chances); + var drafts: [3]u32 = undefined; + try dr.held(&.{&s}, &.{&drafts}); + const rows = r % 3 + 1; + const at: usize = @intCast(anchor % (ctx - 3)); + try dr.absorb(&.{.{ .stream = &s, .features = feats.at(taps, at, rows, width), .start = anchor, .follow = &.{}, .prompt = false }}); + anchor += rows; + } + d.gpa = gpa; + const n = rounds - warm; + std.debug.print("steady rounds: {d} rounds of hold, chances, held and a verified absorb: {d} host allocations, {d} resizes, {d} unwaited holds\n", .{ n, counted.allocations, counted.resize_index, d.signalled - holds }); + if (counted.allocations != 0 or counted.resize_index != 0 or d.signalled - holds != n) return error.SteadyRoundsAllocate; +} + +pub fn main(init: std.process.Init) !void { + const gpa = init.gpa; + const args = try init.minimal.args.toSlice(init.arena.allocator()); + if (args.len < 4 or args.len > 6) return error.ExpectedModelDrafterPromptLengthRowsBits; + const bits: u8 = if (args.len == 6) try std.fmt.parseInt(u8, args[5], 10) else 0; + const length = try std.fmt.parseInt(usize, args[3], 10); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const m = try q.Model.load(gpa, init.io, args[1]); + defer m.deinit(); + const d = try q.dspark.Drafter.init(gpa, init.io, m, args[2], .{ .capacity = length + 64, .bits = bits }); + defer d.deinit(); + const cfg = d.config; + // the compared block's rows (a hold of depth d runs a d-row block) + const block: u32 = if (args.len >= 5) try std.fmt.parseInt(u32, args[4], 10) else cfg.block; + if (block == 0 or block > cfg.block) return error.BadBlockRows; + var scratch = try st.Scratch.init(gpa, m.device, &m.config.attention, length); + defer scratch.deinit(); + const width = cfg.tap_count * c.hidden; + var taps: st.Taps = .{ .count = cfg.tap_count, .buf = try scratch.memory.alloc(length * width * 2) }; + @memcpy(taps.layers[0..cfg.tap_count], cfg.taps[0..cfg.tap_count]); + scratch.taps = taps; + var cache = try st.Cache.init(gpa, m.device, &m.config.attention, length + 16); + defer cache.deinit(); + const ids = try gpa.alloc(u32, length); + defer gpa.free(ids); + for (ids, 0..) |*id, i| id.* = @intCast((i * 7919 + 13) % 120000); + try q.forward.run(m, &scratch, &.{.{ .cache = &cache, .rows = length }}, ids, false, .last); + // the context: rows [0, length - 1); the pending token is the last id, at position length - 1 + const ctx = length - 1; + const lane = try d.newLane(length + 32); + defer d.dropLane(lane); + var at: usize = 0; + while (at < ctx) : (at += q.dspark.absorb_rows) { + const rows = @min(q.dspark.absorb_rows, ctx - at); + try absorb(d, .{ .lane = lane, .src = .{ .buf = taps.buf, .off = at * width * 2 }, .rows = rows, .p0 = at }); + } + lane.end = ctx; + _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null); + var held: [16]u32 = undefined; + @memcpy(held[0..block], lane.held[0..block]); + var conf: [16]f32 = undefined; + @memcpy(conf[0..block], lane.conf[0..block]); + const gpu = held[0..block]; + for ([_]usize{ 1, 4, 16, 128, q.dspark.absorb_rows }) |rows| { + if (rows > ctx) continue; + var best: f64 = std.math.floatMax(f64); + for (0..5) |_| { + const t0 = std.Io.Clock.awake.now(init.io); + try absorb(d, .{ .lane = lane, .src = .{ .buf = taps.buf, .off = (ctx - rows) * width * 2 }, .rows = rows, .p0 = ctx - rows }); + best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6); + } + std.debug.print("absorb of {d} rows: {d:.2} ms (best of 5)\n", .{ rows, best }); + } + const depths = [_]u32{ 1, 3, cfg.block }; + for (depths) |depth| { + var best: f64 = std.math.floatMax(f64); + for (0..5) |_| { + const t0 = std.Io.Clock.awake.now(init.io); + _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = depth }}, null); + best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6); + } + std.debug.print("hold at depth {d}: {d:.2} ms (best of 5)\n", .{ depth, best }); + } + // a shared round's hold: S streams' blocks of d rows in one pass (S x d at most pass_rows), each on this context + var lanes_: [16]*q.dspark.Lane = undefined; + lanes_[0] = lane; + var made: usize = 1; + defer for (lanes_[1..made]) |l| d.dropLane(l); + while (made < lanes_.len) : (made += 1) { + const l = try d.newLane(length + 32); + lanes_[made] = l; + var from: usize = 0; + while (from < ctx) : (from += q.dspark.absorb_rows) { + const rows = @min(q.dspark.absorb_rows, ctx - from); + try absorb(d, .{ .lane = l, .src = .{ .buf = taps.buf, .off = from * width * 2 }, .rows = rows, .p0 = from }); + } + l.end = ctx; + } + for ([_]usize{ 2, 4, 8, 16 }) |streams| for ([_]u32{ 1, 2, 3, 4, 8 }) |depth| { + if (streams * depth > q.dspark.pass_rows) continue; + var blocks: [16]q.dspark_block.Block = undefined; + for (blocks[0..streams], lanes_[0..streams]) |*b, l| b.* = .{ .lane = l, .pending = ids[ctx], .anchor = ctx, .depth = depth }; + var best: f64 = std.math.floatMax(f64); + var gpu_best: f64 = std.math.floatMax(f64); + for (0..5) |_| { + const t0 = std.Io.Clock.awake.now(init.io); + _ = try q.dspark_block.hold(d, blocks[0..streams], null); + best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6); + gpu_best = @min(gpu_best, try gpuHold(d, blocks[0..streams])); + } + std.debug.print("hold of {d} streams at depth {d}: {d:.2} ms, GPU {d:.2} ms (best of 5)\n", .{ streams, depth, best, gpu_best }); + }; + try steady(gpa, d, taps.buf, ids, ctx); + _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null); + // the host reference, fp32 throughout + var r: Ref = .{ .gpa = gpa, .check = ck.Checkpoint.init(gpa), .cfg = cfg }; + defer r.check.deinit(); + const path = try std.fs.path.joinZ(gpa, &.{ args[2], "model.safetensors" }); + defer gpa.free(path); + try r.check.addFile(m.device, path, ""); + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + r.gpa = arena.allocator(); + const a = r.gpa; + const n = block; + const tap_rows = taps.buf.slice(u16, length * width); + const feats = try a.alloc(f32, ctx * width); + for (feats, tap_rows[0 .. ctx * width]) |*f, v| f.* = quant.toF32(v); + const hctx = try a.alloc(f32, ctx * c.hidden); + try matmul(gpa, try r.get("fc.weight"), feats, hctx, c.hidden, width, ctx); + const hn = try r.get("hidden_norm.weight"); + for (0..ctx) |j| rms(hctx[j * c.hidden ..][0..c.hidden], hn); + const h = try a.alloc(f32, n * c.hidden); + for (0..n) |i| embedRow(m, if (i == 0) ids[ctx] else cfg.mask, h[i * c.hidden ..][0..c.hidden]); + const keys = ctx + n; + var name: [96]u8 = undefined; + const x = try a.alloc(f32, n * c.hidden); + for (0..cfg.layers) |li| { + const kmat = try r.layer(&name, li, "self_attn.k_proj"); + const vmat = try r.layer(&name, li, "self_attn.v_proj"); + const qn = try r.layer(&name, li, "self_attn.q_norm"); + const kn = try r.layer(&name, li, "self_attn.k_norm"); + const kk = try a.alloc(f32, keys * kw); + const vv = try a.alloc(f32, keys * kw); + try matmul(gpa, kmat, hctx, kk, kw, c.hidden, ctx); + try matmul(gpa, vmat, hctx, vv, kw, c.hidden, ctx); + @memcpy(x, h); + for (0..n) |i| rms(x[i * c.hidden ..][0..c.hidden], try r.layer(&name, li, "input_layernorm")); + const qq = try a.alloc(f32, n * qw); + try matmul(gpa, try r.layer(&name, li, "self_attn.q_proj"), x, qq, qw, c.hidden, n); + try matmul(gpa, kmat, x, kk[ctx * kw ..], kw, c.hidden, n); + try matmul(gpa, vmat, x, vv[ctx * kw ..], kw, c.hidden, n); + for (0..keys) |j| for (0..c.kv_heads) |kh| { + const row = kk[j * kw + kh * hd ..][0..hd]; + rms(row, kn); + rope(row, j, cfg.theta); + }; + for (0..n) |i| for (0..c.query_heads) |qh| { + const row = qq[i * qw + qh * hd ..][0..hd]; + rms(row, qn); + rope(row, ctx + i, cfg.theta); + }; + const o = try a.alloc(f32, n * qw); + const sc = try a.alloc(f32, keys); + for (0..n) |i| for (0..c.query_heads) |qh| { + const kh = qh / (c.query_heads / c.kv_heads); + var top: f32 = -std.math.inf(f32); + for (0..keys) |j| { + var s: f32 = 0; + for (0..hd) |t| s += qq[i * qw + qh * hd + t] * kk[j * kw + kh * hd + t]; + sc[j] = s / @sqrt(@as(f32, hd)); + top = @max(top, sc[j]); + } + var sum: f32 = 0; + for (sc) |*s| { + s.* = @exp(s.* - top); + sum += s.*; + } + for (0..hd) |t| { + var acc: f32 = 0; + for (0..keys) |j| acc += sc[j] * vv[j * kw + kh * hd + t]; + o[i * qw + qh * hd + t] = acc / sum; + } + }; + const y = try a.alloc(f32, n * c.hidden); + try matmul(gpa, try r.layer(&name, li, "self_attn.o_proj"), o, y, c.hidden, qw, n); + for (h, y) |*v, w| v.* += w; + @memcpy(x, h); + for (0..n) |i| rms(x[i * c.hidden ..][0..c.hidden], try r.layer(&name, li, "post_attention_layernorm")); + const gate = try a.alloc(f32, n * cfg.intermediate); + const up = try a.alloc(f32, n * cfg.intermediate); + try matmul(gpa, try r.layer(&name, li, "mlp.gate_proj"), x, gate, cfg.intermediate, c.hidden, n); + try matmul(gpa, try r.layer(&name, li, "mlp.up_proj"), x, up, cfg.intermediate, c.hidden, n); + for (gate, up) |*g, u| g.* = g.* / (1 + @exp(-g.*)) * u; + try matmul(gpa, try r.layer(&name, li, "mlp.down_proj"), gate, y, c.hidden, cfg.intermediate, n); + for (h, y) |*v, w| v.* += w; + } + const fin = try r.get("norm.weight"); + for (0..n) |i| rms(h[i * c.hidden ..][0..c.hidden], fin); + const head = try a.alloc(f32, c.vocab * c.hidden); + for (0..c.vocab) |v| embedRow(m, @intCast(v), head[v * c.hidden ..][0..c.hidden]); + const base = try a.alloc(f32, n * c.vocab); + try matmul(gpa, head, h, base, c.vocab, c.hidden, n); + const w1 = try r.get("markov_head.markov_w1.weight"); + const w2 = try r.get("markov_head.markov_w2.weight"); + const bias = try a.alloc(f32, c.vocab); + const proj = try r.get("confidence_head.proj.weight"); + const proj_bias = (try r.get("confidence_head.proj.bias"))[0]; + var conf_worst: f32 = 0; + var prev = ids[ctx]; + var same: usize = 0; + var worst: f32 = 0; + for (0..n) |i| { + try matmul(gpa, w2, w1[@as(usize, prev) * cfg.rank ..][0..cfg.rank], bias, c.vocab, cfg.rank, 1); + const row = base[i * c.vocab ..][0..c.vocab]; + var best: usize = 0; + var second: usize = 1; + for (row, bias, 0..) |b, m2, v| { + const s = b + m2; + if (s > row[best] + bias[best]) { + second = best; + best = v; + } else if (v != best and s > row[second] + bias[second]) second = v; + } + const margin = (row[best] + bias[best]) - (row[second] + bias[second]); + if (i == 0) { // the GPU keeps the base logits (the Markov step adds its bias on the way to the pick) + const zero = try a.alloc(f32, c.vocab); + @memset(zero, 0); + compare("row 0 base logits", d.s.logits.slice(u16, c.vocab), row, zero); + } + const ok = gpu[i] == best; + same += @intFromBool(ok); + std.debug.print("step {d}: reference {d} (margin {d:.3}, second {d}), GPU {d} {s}\n", .{ i, best, margin, second, gpu[i], if (ok) "equal" else "DIFFERENT" }); + if (!ok) worst = @max(worst, margin); + // the confidence head: [final norm row i, W1 of the token before draft i] . proj + bias, through a sigmoid + var logit: f32 = proj_bias; + for (h[i * c.hidden ..][0..c.hidden], proj[0..c.hidden]) |x0, w0| logit += x0 * w0; + for (w1[@as(usize, prev) * cfg.rank ..][0..cfg.rank], proj[c.hidden..]) |x0, w0| logit += x0 * w0; + const want = 1 / (1 + @exp(-logit)); + conf_worst = @max(conf_worst, @abs(want - conf[i])); + std.debug.print("step {d}: confidence reference {d:.4}, GPU {d:.4}\n", .{ i, want, conf[i] }); + prev = gpu[i]; // teacher-forced: step i + 1 follows the GPU's pick, so each step is compared alone + } + std.debug.print("the reference's largest margin over a different GPU pick: {d:.3}\n", .{worst}); + std.debug.print("the confidences' worst difference from the fp32 reference: {d:.5}\n", .{conf_worst}); + std.debug.print("{d} of {d} drafts equal the fp32 reference's picks on the same chain ({d}-token context)\n", .{ same, n, ctx }); +}