From b27b3ec1fae427f0964c95554537dc33e68a2b60 Mon Sep 17 00:00:00 2001 From: JRB AI <197533392+jrb-ai@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:36:57 +1100 Subject: [PATCH 1/5] flashnext cuda: a shared round's sampled streams draft in the batched head pass A stream with a temperature above zero was left out of the shared round's batched MTP head pass and drafted alone, one head forward per draft token. Now it joins the pass: each step, after the shared best-column download, a sampled lane's score row is downloaded and its draft drawn on the host under the lane's own keyed rule at the drafted position, the draw its lone pick makes, so drafts, probabilities, the confidence stop and replies are unchanged. Greedy lanes keep their best column. TF_FLASHNEXT_SAMPLED_HEADS=0 restores the previous behaviour; TF_FLASHNEXT_BATCH_HEADS=0 still drafts every stream alone. The parity checker now also runs sampled and mixed shapes, each lane under its own seed and position, at confidence 0 and 0.5. --- zig/src/families/flashnext/cuda_lanes.zig | 4 +- .../families/flashnext/cuda_lanes_draft.zig | 20 ++++++--- zig/src/families/flashnext/cuda_mtp.zig | 38 ++++++++++++++++ zig/src/families/flashnext/cuda_mtp_ops.zig | 19 ++++---- zig/tests/cuda/flashnext_mtp_parity_tests.zig | 45 +++++++++++++++---- 5 files changed, 101 insertions(+), 25 deletions(-) diff --git a/zig/src/families/flashnext/cuda_lanes.zig b/zig/src/families/flashnext/cuda_lanes.zig index 5fe5814d7d..bd9ff8522b 100644 --- a/zig/src/families/flashnext/cuda_lanes.zig +++ b/zig/src/families/flashnext/cuda_lanes.zig @@ -222,7 +222,7 @@ pub fn Namespace(comptime Tri: type) type { .hidden_rows = true, .batch_rows = if (drafting) mtp.round_rows else round_streams, .max_streams = round_streams, - // a shared round's greedy streams draft in one head pass a step (lanes_draft.batched) + // a shared round's streams draft in one head pass a step (lanes_draft.batched) .draft_streams = drafting and lanes_draft.batchHeads(), }; } @@ -488,7 +488,7 @@ pub fn Namespace(comptime Tri: type) type { const self = of(ptr); if (self.head == null) return error.NoDraftHead; defer lanes_draft.depthLog(self, requests); - // greedy chains of a shared round: their heads in one pass a step (the drafts each would get alone) + // a shared round's chains: their heads in one pass a step (the drafts each would get alone) var batched: [mtp_ops.max_lanes]bool = @splat(false); if (lanes_draft.batchHeads() and requests.len > 1 and requests.len <= mtp_ops.max_lanes) try lanes_draft.batched(self, requests, &batched); for (requests, 0..) |r, ri| { diff --git a/zig/src/families/flashnext/cuda_lanes_draft.zig b/zig/src/families/flashnext/cuda_lanes_draft.zig index 1d6bae4fe3..6f0474d22b 100644 --- a/zig/src/families/flashnext/cuda_lanes_draft.zig +++ b/zig/src/families/flashnext/cuda_lanes_draft.zig @@ -1,4 +1,4 @@ -//! A shared round's greedy streams drafted in one head pass a step (cuda_lanes.zig's draftFn), and the depth log. +//! A shared round's streams drafted in one head pass a step (cuda_lanes.zig's draftFn), and the depth log. const std = @import("std"); const lanes = @import("lanes"); const mtp = @import("cuda_mtp.zig"); @@ -11,24 +11,32 @@ pub fn batchHeads() bool { return !std.mem.eql(u8, std.mem.span(v), "0"); } +/// TF_FLASHNEXT_SAMPLED_HEADS=0 keeps sampled streams (temperature above zero) out of a shared round's batched head. +pub fn sampledHeads() bool { + const v = std.c.getenv("TF_FLASHNEXT_SAMPLED_HEADS") orelse return true; + return !std.mem.eql(u8, std.mem.span(v), "0"); +} + /// TF_FLASHNEXT_DEPTH_LOG=1: every 200 draft calls, the drafts a stream got by round width (streams drafting together). var depth_hist: [5][max_hist]u64 = @splat(@splat(0)); var depth_calls: u64 = 0; const max_hist = 17; -/// Greedy requests (no trees, no media) drafted together: draftFn's rollbacks, one head pass a step; `done` marks them. +/// Requests without trees or media drafted together: draftFn's rollbacks, one head pass a step; `done` marks them. pub fn batched(self: anytype, requests: []const be.DraftRequest, done: []bool) !void { var ds: [mtp_ops.max_lanes]*mtp.Draft = undefined; var nexts: [mtp_ops.max_lanes][]const u32 = undefined; var firsts: [mtp_ops.max_lanes][1]u32 = undefined; var rows: [mtp_ops.max_lanes]u64 = undefined; var counts: [mtp_ops.max_lanes]usize = undefined; + var positions: [mtp_ops.max_lanes]u64 = undefined; + var sms: [mtp_ops.max_lanes]?mtp.Sampling = undefined; var outs: [mtp_ops.max_lanes]*mtp.Chain = undefined; var which: [mtp_ops.max_lanes]usize = undefined; var n: usize = 0; for (requests, 0..) |r, ri| { if (r.lanes != null) continue; - if (r.stream.sampling) |sm| if (sm.temperature > 0) continue; + if (mtp.isSampled(r.stream.sampling) and !sampledHeads()) continue; const l = try self.lane(r.stream); const d = if (l.draft) |*x| x else continue; if (d.media != null) continue; @@ -51,15 +59,17 @@ pub fn batched(self: anytype, requests: []const be.DraftRequest, done: []bool) ! rows[n] = d.prompt_last; } ds[n] = d; + positions[n] = r.position; + sms[n] = r.stream.sampling; outs[n] = &l.chain; which[n] = ri; n += 1; } if (n < 2) { - // one greedy stream: it drafts alone below (its rollback is idempotent) + // one stream: it drafts alone below (its rollback is idempotent) return; } - try self.ops().draftMany(ds[0..n], nexts[0..n], rows[0..n], counts[0..n], self.confidence, outs[0..n]); + try self.ops().draftMany(ds[0..n], nexts[0..n], rows[0..n], counts[0..n], positions[0..n], sms[0..n], self.confidence, outs[0..n]); for (which[0..n]) |ri| done[ri] = true; } diff --git a/zig/src/families/flashnext/cuda_mtp.zig b/zig/src/families/flashnext/cuda_mtp.zig index 2d822aa88b..8fe177ec9a 100644 --- a/zig/src/families/flashnext/cuda_mtp.zig +++ b/zig/src/families/flashnext/cuda_mtp.zig @@ -132,6 +132,22 @@ pub fn drawRow(gpa: std.mem.Allocator, row: []const u16, ids: ?[]const u32, posi return @intCast(try lanes.sampling.choose(gpa, vals, keys, position, s)); } +/// Whether a stream's rule draws its drafts (a temperature above zero) rather than taking each row's best column. +pub fn isSampled(sm: ?Sampling) bool { + return if (sm) |s| s.temperature > 0 else false; +} + +/// A draft drawn from a downloaded score row (column j is token ids[j]) and its probability under the row's lse. +pub fn drawPick(gpa: std.mem.Allocator, row: []const u16, ids: []const u32, lse: f32, position: u64, sm: ?Sampling) !Pick { + const tok = try drawRow(gpa, row, ids, position, sm); + const col = std.sort.binarySearch(u32, ids, tok, struct { + fn order(t: u32, x: u32) std.math.Order { + return std.math.order(t, x); + } + }.order) orelse return .{ .token = tok, .prob = 0 }; + return .{ .token = tok, .prob = @exp(@as(f64, promote(row[col])) - @as(f64, lse)) }; +} + // ---- the head's weights -------------------------------------------------------------------------------------------- pub const Head = struct { @@ -288,6 +304,28 @@ test "a keyed draw over the top-k columns is the draw over every column" { try std.testing.expectEqual(@as(u32, @intCast(argmaxRow(&row))), try drawRow(gpa, &row, null, 0, null)); } +test "a drawn pick is the row's keyed draw with its probability under the row's log-sum-exp" { + const gpa = std.testing.allocator; + var row: [300]u16 = undefined; + var prng = std.Random.DefaultPrng.init(7); + for (&row) |*v| v.* = @as(u16, 0x3f00) + prng.random().int(u8); + var ids: [300]u32 = undefined; + for (&ids, 0..) |*t, i| t.* = @intCast(3 * i + 1); // a sorted map of columns to tokens, as the draft head's + var sum: f64 = 0; + for (row) |b| sum += @exp(@as(f64, promote(b))); + const lse: f32 = @floatCast(@log(sum)); + for (0..32) |position| { + const s: Sampling = .{ .seed = 19, .temperature = 1.0, .top_k = 20, .top_p = 0.95 }; + const got = try drawPick(gpa, &row, &ids, lse, position, s); + try std.testing.expectEqual(try drawRow(gpa, &row, &ids, position, s), got.token); + const col = (got.token - 1) / 3; + try std.testing.expectEqual(@exp(@as(f64, promote(row[col])) - @as(f64, lse)), got.prob); + } + const best = try drawPick(gpa, &row, &ids, lse, 0, null); + try std.testing.expectEqual(@as(u32, @intCast(3 * argmaxRow(&row) + 1)), best.token); + try std.testing.expect(best.prob > 0 and best.prob <= 1); +} + test "a masked row draws only allowed tokens, greedy the best of them" { const gpa = std.testing.allocator; var prng = std.Random.DefaultPrng.init(5); diff --git a/zig/src/families/flashnext/cuda_mtp_ops.zig b/zig/src/families/flashnext/cuda_mtp_ops.zig index 3366bf7c8f..993e5328ce 100644 --- a/zig/src/families/flashnext/cuda_mtp_ops.zig +++ b/zig/src/families/flashnext/cuda_mtp_ops.zig @@ -210,13 +210,7 @@ pub fn Ops(comptime Tri: type) type { } const row = d.host[0..n]; try download(o.stream(), std.mem.sliceAsBytes(row), d.dlogits.ptr); - const tok = try drawRow(o.e.gpa, row, h.ids, position, sm); - const col = std.sort.binarySearch(u32, h.ids, tok, struct { - fn order(t: u32, x: u32) std.math.Order { - return std.math.order(t, x); - } - }.order) orelse return .{ .token = tok, .prob = 0 }; - return .{ .token = tok, .prob = @exp(@as(f64, promote(row[col])) - @as(f64, lse)) }; + return mtp.drawPick(o.e.gpa, row, h.ids, lse, position, sm); } /// Python decode.draft: absorb the kept rows, then chain up to `count` drafts, the first always kept. @@ -350,8 +344,8 @@ pub fn Ops(comptime Tri: type) type { for (ds, 0..) |d, i| try p.copy(d.dlogits.ptr, s.head_local + i * n_ids * 2, n_ids * 2); } - /// draft over several greedy drafts: kept rows in one pass, then a pass a step, each chaining as it would alone - pub fn draftMany(o: Self, ds: []const *Draft, nexts: []const []const u32, rows: []const u64, counts: []const usize, confidence: f64, outs: []const *Chain) !void { + /// draft over several drafts, greedy or sampled: kept rows in one pass, then one a step, each as it would alone + pub fn draftMany(o: Self, ds: []const *Draft, nexts: []const []const u32, rows: []const u64, counts: []const usize, positions: []const u64, sms: []const ?Sampling, confidence: f64, outs: []const *Chain) !void { const h = o.h; const n_l = ds.len; for (ds, outs) |d, out| { @@ -384,7 +378,12 @@ pub fn Ops(comptime Tri: type) type { const top = best_host[r * 4]; const lse = best_host[r * 4 + 1]; const col: usize = @intCast(@as(i32, @bitCast(best_host[r * 4 + 2]))); - const got: Pick = .{ .token = h.ids[col], .prob = @exp(@as(f64, top) - @as(f64, lse)) }; + // a sampled lane draws from its own row, as its lone pick does; a greedy one takes its best column + const got: Pick = if (mtp.isSampled(sms[i])) sampled: { + const row = ds[i].host[0..h.view().logical]; + try download(o.stream(), std.mem.sliceAsBytes(row), ds[i].dlogits.ptr); + break :sampled try mtp.drawPick(o.e.gpa, row, h.ids, lse, positions[i] + j, sms[i]); + } else .{ .token = h.ids[col], .prob = @exp(@as(f64, top) - @as(f64, lse)) }; const out = outs[i]; const low = confidence > 0 and got.prob < confidence; if (low and j > 0) continue; diff --git a/zig/tests/cuda/flashnext_mtp_parity_tests.zig b/zig/tests/cuda/flashnext_mtp_parity_tests.zig index d3b4ee863c..f534d5e03b 100644 --- a/zig/tests/cuda/flashnext_mtp_parity_tests.zig +++ b/zig/tests/cuda/flashnext_mtp_parity_tests.zig @@ -1,4 +1,4 @@ -//! A shared round's greedy streams' MTP heads in one pass against each stream's own pass, bit for bit. +//! A shared round's streams' MTP heads in one pass against each stream's own pass, bit for bit: greedy, sampled, mixed. const std = @import("std"); const cuda = @import("cuda"); const flash = @import("flashnext"); @@ -9,6 +9,24 @@ const mtp = flash.mtp; /// The streams' rows in one shared pass: two one-row heads, four 16-row paths, and both together. const shapes = [_][]const usize{ &.{ 1, 1 }, &.{ 16, 16, 16, 16 }, &.{ 1, 1, 16, 16, 16, 16 } }; +/// How a shape's streams draw: every stream greedy, every stream sampled under its own seed, or alternate lanes. +const Mode = enum { greedy, sampled, mixed }; + +/// Stream i's rule: greedy (null), or a keyed draw under its own seed. +fn rule(mode: Mode, i: usize) ?mtp.Sampling { + const sampled = switch (mode) { + .greedy => false, + .sampled => true, + .mixed => i % 2 == 1, + }; + return if (sampled) .{ .seed = 19 + @as(u64, i), .temperature = 1.0, .top_k = 20, .top_p = 0.95 } else null; +} + +/// Stream i's first draft position: the greedy shapes keep position 0, the others each their own. +fn position(mode: Mode, i: usize) u64 { + return if (mode == .greedy) 0 else 5 + 3 * @as(u64, i); +} + /// The streams one side holds: a sequence and a draft each. const Side = struct { seqs: [6]prompt.Seq = undefined, @@ -55,15 +73,22 @@ pub fn run(gpu: check.Gpu, dir: []const u8, ids_path: []const u8) !void { for (ids, 0..) |*t, i| t.* = base.items[i % base.items.len]; // no confidence stop (the whole chain), then a stop above every probability (each chain keeps one token) for ([_]f64{ 0, 2 }) |confidence| { - for (shapes) |widths| try shape(gpu, e, ops, &stream, ids, widths, confidence); + for (shapes) |widths| try shape(gpu, e, ops, &stream, ids, widths, confidence, .greedy); + } + // sampled streams under their own seeds and positions, alone and beside greedy ones; a stop some draws fall under + for ([_]f64{ 0, 0.5 }) |confidence| { + for (shapes) |widths| { + try shape(gpu, e, ops, &stream, ids, widths, confidence, .sampled); + try shape(gpu, e, ops, &stream, ids, widths, confidence, .mixed); + } } } /// One shape's streams drafted alone and in one shared pass: the logits, the proposals, the probabilities and the stop. -fn shape(gpu: check.Gpu, e: *prompt.Engine, ops: mtp.Ops(flash.Tri), stream: *cuda.Stream, ids: []const u32, widths: []const usize, confidence: f64) !void { +fn shape(gpu: check.Gpu, e: *prompt.Engine, ops: mtp.Ops(flash.Tri), stream: *cuda.Stream, ids: []const u32, widths: []const usize, confidence: f64, mode: Mode) !void { const gpa = gpu.gpa; const cap = ids.len + 64; - const count = 2; + const count: usize = if (mode == .greedy) 2 else 4; const n = widths.len; var alone: Side = .{}; defer alone.deinit(gpa); @@ -86,16 +111,18 @@ fn shape(gpu: check.Gpu, e: *prompt.Engine, ops: mtp.Ops(flash.Tri), stream: *cu } rows[i] = alone.drafts[i].prompt_last; } - // each stream's head alone: its own pass, its own chain + // each stream's head alone: its own pass, its own chain, its own rule and position var alone_chains: [6]mtp.Chain = undefined; for (widths, 0..) |w, i| { - try ops.draft(&alone.drafts[i], nexts[i][0..w], rows[i], 0, count, null, confidence, &alone_chains[i]); + try ops.draft(&alone.drafts[i], nexts[i][0..w], rows[i], position(mode, i), count, rule(mode, i), confidence, &alone_chains[i]); } // the same heads in one shared pass: one lane a stream var ds: [6]*mtp.Draft = undefined; var nexts_s: [6][]const u32 = undefined; var rows_s: [6]u64 = undefined; var counts: [6]usize = undefined; + var positions: [6]u64 = undefined; + var sms: [6]?mtp.Sampling = undefined; var chains: [6]mtp.Chain = undefined; var outs: [6]*mtp.Chain = undefined; for (widths, 0..) |w, i| { @@ -103,9 +130,11 @@ fn shape(gpu: check.Gpu, e: *prompt.Engine, ops: mtp.Ops(flash.Tri), stream: *cu nexts_s[i] = nexts[i][0..w]; rows_s[i] = rows[i]; counts[i] = count; + positions[i] = position(mode, i); + sms[i] = rule(mode, i); outs[i] = &chains[i]; } - try ops.draftMany(ds[0..n], nexts_s[0..n], rows_s[0..n], counts[0..n], confidence, outs[0..n]); + try ops.draftMany(ds[0..n], nexts_s[0..n], rows_s[0..n], counts[0..n], positions[0..n], sms[0..n], confidence, outs[0..n]); try stream.synchronize(); for (widths, 0..) |_, i| { try check.expect(try sameLogits(gpu, &alone.drafts[i], &together.drafts[i]), "a shared head's logits equal its lone pass's", .{}); @@ -122,5 +151,5 @@ fn shape(gpu: check.Gpu, e: *prompt.Engine, ops: mtp.Ops(flash.Tri), stream: *cu try stream.synchronize(); try check.expect(!(try sameLogits(gpu, &alone.drafts[1], &together.drafts[1])), "the planted broken gather is caught", .{}); } - check.pass("a shared MTP head over {d} streams ({any} rows) equals each stream's own pass: logits, proposals, probabilities and the confidence stop", .{ n, widths }); + check.pass("a shared MTP head over {d} {s} streams ({any} rows) equals each stream's own pass: logits, proposals, probabilities and the stop", .{ n, @tagName(mode), widths }); } From d596352c74cf74198ec3b69ca92a565d07d8854f Mon Sep 17 00:00:00 2001 From: JRB AI <197533392+jrb-ai@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:43:47 +1100 Subject: [PATCH 2/5] docs: receipt and changelog entry for sampled streams in the batched head pass (GB10) --- CHANGELOG.md | 2 + .../flashnext-cuda-sampled-heads/A1.json | 23728 ++++++++++++++++ .../flashnext-cuda-sampled-heads/A2.json | 23728 ++++++++++++++++ .../flashnext-cuda-sampled-heads/B1.json | 23728 ++++++++++++++++ .../flashnext-cuda-sampled-heads/B2.json | 23728 ++++++++++++++++ .../flashnext-cuda-sampled-heads/README.md | 75 + .../flashnext-cuda-sampled-heads/commit.txt | 1 + .../flashnext-cuda-sampled-heads/parity.txt | 19 + 8 files changed, 95009 insertions(+) create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/A1.json create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/A2.json create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/B1.json create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/B2.json create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/README.md create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/commit.txt create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/parity.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index da2fb76720..6dd58e26eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel ## Unreleased +- Flash Next on CUDA drafts a shared round's sampled streams in the batched MTP head pass too, each lane's draft drawn on the host under its own keyed rule from its own row of the pass, the draw its lone pick makes, so drafts and replies are unchanged. On a GB10 at 5 x 131,072, sampled code and chat requests decode 24% and 19% faster in aggregate at 5 streams (15% and 19% at 4), greedy cells and time to first token unchanged, every concurrent reply equal to its lone run. `TF_FLASHNEXT_SAMPLED_HEADS=0` restores drafting those streams one at a time. The parity checker covers sampled and mixed shapes. Receipt: `docs/receipts/flashnext-cuda-sampled-heads/`. + - Kernel contract: Qwen4's dense activation check carries FP32 rounding through the stored BF16 SiLU before the up product, avoiding a false shape failure near a BF16 midpoint (#1547). Inference kernels are unchanged. - HIP: `-Dhip-relax-branches=true` (off by default) lets gfx1100 build the dot2 prompt tiles with Ubuntu's clang 21, which otherwise stops on a branch out of range (#1542, for #1521). It has not run on a gfx1100 GPU yet; with it off, every compile command is the same as before. Thanks to @AbrahamCete. diff --git a/docs/receipts/flashnext-cuda-sampled-heads/A1.json b/docs/receipts/flashnext-cuda-sampled-heads/A1.json new file mode 100644 index 0000000000..ed58295c5a --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/A1.json @@ -0,0 +1,23728 @@ +{ + "label": "A1", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "code", + "temperature": 1.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.5, + "per_stream_tps": 66.5, + "ttft_s_max": 0.1, + "request": { + "ttft_s": 0.096, + "decode_tps": 66.546, + "prefill_tps": 145.354, + "itl_mean_ms": 15.027, + "itl_p50_ms": 0.0, + "itl_p95_ms": 79.22, + "round_gap_p50_ms": 66.194, + "round_gap_p95_ms": 82.564, + "tokens_per_chunk": 4.414 + }, + "pooled": { + "aggregate_prefill_tps": 145.354, + "itl_mean_ms": 15.027, + "itl_p50_ms": 0.0, + "itl_p95_ms": 79.22, + "round_gap_p50_ms": 66.194, + "round_gap_p95_ms": 82.564, + "tokens_per_chunk": 4.414 + }, + "timing_valid": true, + "end_to_end_tps": 65.16554427805907, + "makespan_s": 3.928456408000784, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.50128726674359, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.93055784399985, + "end_to_end_tps": 65.13070412913373, + "timing_valid": true, + "per_stream_tps": [ + 66.5 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.0958755409992591, + "decode_tps": 66.50128726674359, + "prefill_tps": 146.02264408717326, + "itl_mean_ms": 15.037302901955805, + "itl_p50_ms": 0.0, + "itl_p95_ms": 79.6309699981066, + "round_gap_p50_ms": 67.79513500077883, + "round_gap_p95_ms": 81.8473190010991, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 15.037302901955805, + "itl_p50_ms": 0.0, + "itl_p95_ms": 79.6309699981066, + "round_gap_p50_ms": 67.79513500077883, + "round_gap_p95_ms": 81.8473190010991, + "round_gap_mean_ms": 67.27214456138124, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 146.02264408717326 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 28404.494481123, + "first": 28404.590356664, + "last": 28408.424868904, + "end": 28408.425038967, + "elapsed_s": 3.93055784399985, + "chunk_offsets_s": [ + 0.0958755409992591, + 0.15118843799791648, + 0.2062515439974959, + 0.26634690599894384, + 0.33414204099972267, + 0.3955588069984515, + 0.46427505599785945, + 0.534878561997175, + 0.6104752419996657, + 0.6855964099995617, + 0.7468196989975695, + 0.803608357997291, + 0.8603826489998028, + 0.9170134559972212, + 0.9915767429993139, + 1.071037677997083, + 1.143431412998325, + 1.225278731999424, + 1.3049097019975306, + 1.3768437209982949, + 1.4588389209966408, + 1.5328352709984756, + 1.612816943998041, + 1.692454280997481, + 1.773375953998766, + 1.8421534819972294, + 1.8960319419966254, + 1.952108050998504, + 1.999884563996602, + 2.044469892996858, + 2.0990672609987087, + 2.153388470000209, + 2.2138065559993265, + 2.2913311119991704, + 2.3577964500000235, + 2.4023904360001325, + 2.456116318997374, + 2.5127365619991906, + 2.562851884998963, + 2.6217212800002017, + 2.7022126729971205, + 2.783372480997059, + 2.847708863999287, + 2.8977907709995634, + 2.962227414998779, + 3.0438725329986482, + 3.1259523709995847, + 3.2060584899991227, + 3.283958096999413, + 3.349491389999457, + 3.409065287996782, + 3.476644895999925, + 3.541643986998679, + 3.6220028289972106, + 3.7029792969988193, + 3.780659337997349, + 3.858497453999007, + 3.9303877809979895 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 67.79513500077883, + "p95": 81.8473190010991, + "max": 82.07983800093643 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.54579104767997, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.928456408000784, + "end_to_end_tps": 65.16554427805907, + "timing_valid": true, + "per_stream_tps": [ + 66.5 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09638895400348702, + "decode_tps": 66.54579104767997, + "prefill_tps": 145.24485865354995, + "itl_mean_ms": 15.02724641568242, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.65640899763093, + "round_gap_p50_ms": 66.19428900012281, + "round_gap_p95_ms": 82.56398600133252, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 15.02724641568242, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.65640899763093, + "round_gap_p50_ms": 66.19428900012281, + "round_gap_p95_ms": 82.56398600133252, + "round_gap_mean_ms": 67.22715501752661, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 145.24485865354995 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 28408.425634391, + "first": 28408.522023345, + "last": 28412.353971181, + "end": 28412.354090799, + "elapsed_s": 3.928456408000784, + "chunk_offsets_s": [ + 0.09638895400348702, + 0.15228724100234103, + 0.20491644300273038, + 0.26451765000092564, + 0.33071193900104845, + 0.3937169270029699, + 0.45811276200038265, + 0.531592791001458, + 0.6099532810003438, + 0.6862538260029396, + 0.7481248850017437, + 0.8068169340003806, + 0.8624772800030769, + 0.9200250690009852, + 0.9937612680005259, + 1.0737182090015267, + 1.1467787280016637, + 1.228271283001959, + 1.3093182310003613, + 1.3805702390018268, + 1.4583393500033708, + 1.5333275550001417, + 1.6163361960025213, + 1.6954761420020077, + 1.7761539130005985, + 1.8449276410028688, + 1.8990314930015302, + 1.9551590030023362, + 2.003314807003335, + 2.0474139310026658, + 2.1017648260021815, + 2.155863144002069, + 2.216117154002859, + 2.2931975980027346, + 2.359450188003393, + 2.4041610400017817, + 2.452747161001753, + 2.5088990710028156, + 2.5625171910032805, + 2.6250276120008493, + 2.706164915001864, + 2.786534698003379, + 2.850195546001487, + 2.900874846000079, + 2.9644256420033344, + 3.0430820510009653, + 3.1286756890003744, + 3.2091111980007554, + 3.285522124002455, + 3.351062735000596, + 3.4105823780009814, + 3.4722185200007516, + 3.537887780999881, + 3.6204517670012137, + 3.700427611001942, + 3.777743788999942, + 3.8562633440014906, + 3.928336790002504 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 66.19428900012281, + "p95": 82.56398600133252, + "max": 85.59363799940911 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.82440936249444, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.9123583159998816, + "end_to_end_tps": 65.4336794646515, + "timing_valid": true, + "per_stream_tps": [ + 66.8 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09631667000212474, + "decode_tps": 66.82440936249444, + "prefill_tps": 145.3538624174939, + "itl_mean_ms": 14.964591674509517, + "itl_p50_ms": 0.0, + "itl_p95_ms": 79.21992699994007, + "round_gap_p50_ms": 65.40616599886562, + "round_gap_p95_ms": 83.64574200095376, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.964591674509517, + "itl_p50_ms": 0.0, + "itl_p95_ms": 79.21992699994007, + "round_gap_p50_ms": 65.40616599886562, + "round_gap_p95_ms": 83.64574200095376, + "round_gap_mean_ms": 66.9468574912268, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 145.3538624174939 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 28412.354728338, + "first": 28412.451045008, + "last": 28416.267015885, + "end": 28416.267086654, + "elapsed_s": 3.9123583159998816, + "chunk_offsets_s": [ + 0.09631667000212474, + 0.15157366200219258, + 0.20509400900118635, + 0.26428822000161745, + 0.3301013310010603, + 0.39167182999881334, + 0.4542476490023546, + 0.5237223510011972, + 0.6030278059988632, + 0.6822477329988033, + 0.7440152410017618, + 0.8022014250018401, + 0.8577550959998916, + 0.9144063880012254, + 0.9875996100017801, + 1.0648501840005338, + 1.138554015000409, + 1.221032641002239, + 1.3011815120007668, + 1.3705878399996436, + 1.4476742890001333, + 1.5185754920021282, + 1.602070282002387, + 1.685240397000598, + 1.7666790580005909, + 1.8351231640008336, + 1.8893684840004425, + 1.944573829001456, + 1.9916016810020665, + 2.0361796770012006, + 2.0902165260013135, + 2.144388202999835, + 2.2050541620010335, + 2.2822693699999945, + 2.34767553599886, + 2.392112123001425, + 2.4396817480010213, + 2.491860568999982, + 2.54244080799981, + 2.60662153299927, + 2.69165319899912, + 2.7728878750021977, + 2.8351855680011795, + 2.886508629999298, + 2.9507415939988277, + 3.0292986250024114, + 3.1111462879998726, + 3.192994737000845, + 3.2693673700014187, + 3.334040345998801, + 3.3939786920018378, + 3.4560805030014308, + 3.517647280001256, + 3.60129302200221, + 3.68549091100067, + 3.7630057000023953, + 3.8408689280004182, + 3.9122875470020517 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 65.40616599886562, + "p95": 83.64574200095376, + "max": 85.03166599984979 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 77.5, + "per_stream_tps": 44.0, + "ttft_s_max": 0.19, + "request": { + "ttft_s": 0.144, + "decode_tps": 44.077, + "prefill_tps": 107.483, + "itl_mean_ms": 22.934, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.005, + "round_gap_p50_ms": 84.32, + "round_gap_p95_ms": 109.582, + "tokens_per_chunk": 3.73 + }, + "pooled": { + "aggregate_prefill_tps": 147.458, + "itl_mean_ms": 22.93, + "itl_p50_ms": 0.0, + "itl_p95_ms": 96.498, + "round_gap_p50_ms": 85.359, + "round_gap_p95_ms": 112.488, + "tokens_per_chunk": 3.631 + }, + "timing_valid": true, + "end_to_end_tps": 76.66038998444671, + "makespan_s": 6.678807661999599, + "steady_tps": 85.0, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 77.09440246482197, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.710829786999966, + "end_to_end_tps": 76.29458893322436, + "timing_valid": true, + "steady_tps": 84.40934800849259, + "per_stream_tps": [ + 48.6, + 39.1 + ], + "ttft_s": [ + 0.1, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09523910499774502, + "decode_tps": 48.64503433650002, + "prefill_tps": 146.99844145250503, + "itl_mean_ms": 20.557082827458633, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.33173300305498, + "round_gap_p50_ms": 89.0700290001405, + "round_gap_p95_ms": 114.07236000013654, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.1888725790013268, + "decode_tps": 39.100062234276834, + "prefill_tps": 74.12404740818228, + "itl_mean_ms": 25.57540686273783, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.5113809981849, + "round_gap_p50_ms": 81.8864829998347, + "round_gap_p95_ms": 108.04541800098377, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 23.066244845098232, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.30102699925192, + "round_gap_p50_ms": 86.93033300005482, + "round_gap_p95_ms": 112.48778500157641, + "round_gap_mean_ms": 84.63154583453309, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 148.24809481636456 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 28416.268615321, + "first": 28416.363854426, + "last": 28421.605910547, + "end": 28421.606061579, + "elapsed_s": 5.337446257999545, + "chunk_offsets_s": [ + 0.09523910499774502, + 0.2541759519990592, + 0.318442800999037, + 0.38718510899707326, + 0.4675537980001536, + 0.5402171030000318, + 0.6182120109988318, + 0.7113357199996244, + 0.8069009489991004, + 0.9112715800001752, + 0.9972798529997817, + 1.067523011999583, + 1.1503102549977484, + 1.223929206997127, + 1.3126718289968267, + 1.4010526679994655, + 1.494987340996886, + 1.585431310999411, + 1.6802357729975483, + 1.7677840429969365, + 1.8615836149983807, + 1.949585397996998, + 2.0471353590000945, + 2.136205388000235, + 2.234506414999487, + 2.328640517000167, + 2.4003326069978357, + 2.4770899879986246, + 2.550624132996745, + 2.6206518819999474, + 2.7041475429978163, + 2.7833636939976714, + 2.8690519179981493, + 2.9745231259985303, + 3.06302556999799, + 3.126631377999729, + 3.1865444779978134, + 3.2646438899973873, + 3.356266283997684, + 3.450863248999667, + 3.537657673998183, + 3.6345937639998738, + 3.742440791997069, + 3.841772525000124, + 3.9496776579981088, + 4.063750017998245, + 4.170381696996628, + 4.260773943999084, + 4.353812557998026, + 4.44415522700001, + 4.509383156997501, + 4.591039876999275, + 4.677320859998872, + 4.774148352997145, + 4.883902779998607, + 4.99641506899934, + 5.127497474997654, + 5.23163907399794, + 5.337295225999696 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 89.70634900106234, + "p95": 114.07236000013654, + "max": 158.93684700131416 + } + }, + { + "start": 28416.268519364, + "first": 28416.457391943, + "last": 28422.979120693, + "end": 28422.979349151, + "elapsed_s": 6.710829786999966, + "chunk_offsets_s": [ + 0.1888725790013268, + 0.2545685270015383, + 0.3186836810018576, + 0.38717606000136584, + 0.4679019950017391, + 0.5402158940014488, + 0.6185540430014953, + 0.7112622540007578, + 0.8072341590013821, + 0.9110799520021828, + 0.9976106089998211, + 1.0680015220023051, + 1.1501805070001865, + 1.2242061200013268, + 1.312937963000877, + 1.4012185079991468, + 1.4948762300009548, + 1.5857557720009936, + 1.6801375249997363, + 1.7676518160005799, + 1.8614453590016637, + 1.9501273470013984, + 2.0466213780018734, + 2.136594055002206, + 2.234105436000391, + 2.328623265002534, + 2.4002165439997043, + 2.477230492000672, + 2.550668964002398, + 2.620830660001957, + 2.7043024490012613, + 2.7833593390023452, + 2.8690761159996327, + 2.9745115730002, + 3.063050602002477, + 3.126654165000218, + 3.186573902999953, + 3.2646561450019362, + 3.356294859000627, + 3.4508826689998386, + 3.5378130019998935, + 3.6347792750020744, + 3.74248066400105, + 3.8418065139994724, + 3.949851932000456, + 4.063773492001928, + 4.170422707000398, + 4.260736812000687, + 4.35382134899919, + 4.444185063999612, + 4.509418060002645, + 4.5913045430024795, + 4.677462364001258, + 4.77462219500012, + 4.883937003000028, + 4.996424788001605, + 5.127222134000476, + 5.231967092000559, + 5.337891798000783, + 5.3942723229993135, + 5.442810835000273, + 5.487634053002694, + 5.5432493970001815, + 5.605194763000327, + 5.667000110999652, + 5.7294971690025704, + 5.8107237860022, + 5.87130757200066, + 5.916032429002371, + 5.961698299001, + 6.008028510001168, + 6.070362523001677, + 6.147972623002715, + 6.225757784999587, + 6.299128528000438, + 6.37291266000102, + 6.4494320820012945, + 6.513768749999144, + 6.558832736001932, + 6.605494809002266, + 6.653738995002641, + 6.7106013289994735 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 81.8864829998347, + "p95": 108.04541800098377, + "max": 130.7973459988716 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 77.51910583573172, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.678807661999599, + "end_to_end_tps": 76.66038998444671, + "timing_valid": true, + "steady_tps": 84.9577906689817, + "per_stream_tps": [ + 48.9, + 39.3 + ], + "ttft_s": [ + 0.1, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09940279900183668, + "decode_tps": 48.93839853138069, + "prefill_tps": 140.8411044817895, + "itl_mean_ms": 20.433852149018968, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.50308400200447, + "round_gap_p50_ms": 90.23308500036364, + "round_gap_p95_ms": 121.18681499850936, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19485762100157444, + "decode_tps": 39.32955635122062, + "prefill_tps": 71.84733103093197, + "itl_mean_ms": 25.426170360779174, + "itl_p50_ms": 0.0, + "itl_p95_ms": 96.49773300043307, + "round_gap_p50_ms": 80.60840599864605, + "round_gap_p95_ms": 109.0695519997098, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 22.93001125489907, + "itl_p50_ms": 0.0, + "itl_p95_ms": 96.49773300043307, + "round_gap_p50_ms": 85.35863900033291, + "round_gap_p95_ms": 115.99174900038633, + "round_gap_mean_ms": 84.13169597121242, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 143.69466206186394 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 28422.980654516, + "first": 28423.080057315, + "last": 28428.290689613, + "end": 28428.290924299, + "elapsed_s": 5.310269782999967, + "chunk_offsets_s": [ + 0.09940279900183668, + 0.2604142560012406, + 0.3242755989995203, + 0.39321199000187335, + 0.4738606589999108, + 0.5492544330008968, + 0.6273406319996866, + 0.7141952500023763, + 0.8055128100022557, + 0.9095191619999241, + 1.0018645070013008, + 1.0741400130027614, + 1.1589967040017655, + 1.2306417639993015, + 1.3168227880014456, + 1.4055215139997017, + 1.4977545729998383, + 1.588380909000989, + 1.678389276999951, + 1.7628083100025833, + 1.8545318510005018, + 1.9441426350022084, + 2.047933767000359, + 2.138813300000038, + 2.2373163840020425, + 2.331756054001744, + 2.4032299690006766, + 2.4802984340021794, + 2.555304356999841, + 2.6265130810024857, + 2.704513966000377, + 2.7776003610015323, + 2.861914263001381, + 2.9718935800010513, + 3.064552530999208, + 3.12974283800213, + 3.190658995001286, + 3.26641930400001, + 3.3582281000017247, + 3.451490138999361, + 3.5357877130009, + 3.6319800420023967, + 3.732971155000996, + 3.8289270010027394, + 3.9375356410018867, + 4.058722456000396, + 4.165890596999816, + 4.257084064000082, + 4.351209575001121, + 4.441442660001485, + 4.506065528999898, + 4.589823250000336, + 4.672161148002488, + 4.764926304000255, + 4.873654792001616, + 4.990000252000755, + 5.112263651000831, + 5.213648795001063, + 5.310035097001673 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 90.42971050075721, + "p95": 121.18681499850936, + "max": 161.0114569994039 + } + }, + { + "start": 28422.980549507, + "first": 28423.175407128, + "last": 28429.65908057, + "end": 28429.659357169, + "elapsed_s": 6.678807661999599, + "chunk_offsets_s": [ + 0.19485762100157444, + 0.2608603590015264, + 0.3246641500008991, + 0.393576236001536, + 0.47418464200018207, + 0.5495533249995788, + 0.6277307689997542, + 0.7140797800020664, + 0.8059093080009916, + 0.9097944349996396, + 1.001734690999001, + 1.0739649550014292, + 1.159323594001762, + 1.2310539140016772, + 1.317119592000381, + 1.405914983999537, + 1.4983448530001624, + 1.5888061789992207, + 1.678793241000676, + 1.763187618998927, + 1.854935833998752, + 1.9445656159987266, + 2.0478326459997334, + 2.1391753620009695, + 2.2377066549997835, + 2.332026504998794, + 2.4039150009994046, + 2.4805908470007125, + 2.555717518000165, + 2.6268931000013254, + 2.7048796480012243, + 2.7779912410005636, + 2.862228812002286, + 2.9721626390019082, + 3.06474595599866, + 3.129637955000362, + 3.190602372000285, + 3.2663615130004473, + 3.358567138999206, + 3.451661091999995, + 3.535634730000311, + 3.632132463000744, + 3.733143420999113, + 3.8290919739993114, + 3.9377436989998387, + 4.058753711000463, + 4.166046570000617, + 4.2569252050016075, + 4.351491852001345, + 4.4418464039990795, + 4.506116403001215, + 4.5898715290022665, + 4.672409245999006, + 4.764955123999243, + 4.874024675998953, + 4.9900164249993395, + 5.112664798001788, + 5.213972363999346, + 5.310309866999887, + 5.367399271999602, + 5.416340173000208, + 5.4615092530002585, + 5.516048204000981, + 5.577973801999178, + 5.637068054998963, + 5.696318409001833, + 5.7743390960022225, + 5.835247140999854, + 5.880527461999009, + 5.924836578000395, + 5.9765327099994465, + 6.041115152998827, + 6.120665014001133, + 6.198828469001455, + 6.270714572001452, + 6.345263822000561, + 6.421366395999939, + 6.484924307998881, + 6.5316785690010875, + 6.578677785000764, + 6.626296899001318, + 6.678531063000264 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 80.60840599864605, + "p95": 109.0695519997098, + "max": 122.64837300244835 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 79.0169541390512, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.549754564999603, + "end_to_end_tps": 78.17086807130323, + "timing_valid": true, + "steady_tps": 86.78440314134366, + "per_stream_tps": [ + 50.9, + 39.5 + ], + "ttft_s": [ + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18977094599904376, + "decode_tps": 50.91445580212611, + "prefill_tps": 73.77314755057681, + "itl_mean_ms": 19.640787360791972, + "itl_p50_ms": 0.0, + "itl_p95_ms": 95.612881003035, + "round_gap_p50_ms": 86.75371400022414, + "round_gap_p95_ms": 110.09535099947243, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09533535499940626, + "decode_tps": 39.5084770695256, + "prefill_tps": 146.85003270913703, + "itl_mean_ms": 25.311023713726957, + "itl_p50_ms": 0.0, + "itl_p95_ms": 95.47027399821673, + "round_gap_p50_ms": 79.58680400042795, + "round_gap_p95_ms": 108.6373280013504, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 22.475905537259464, + "itl_p50_ms": 0.0, + "itl_p95_ms": 95.47027399821673, + "round_gap_p50_ms": 84.10339799957, + "round_gap_p95_ms": 110.09535099947243, + "round_gap_mean_ms": 82.46555269066423, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 147.4575723499134 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 28429.660639162, + "first": 28429.850410108, + "last": 28434.858810885, + "end": 28434.858991119, + "elapsed_s": 5.198351956998522, + "chunk_offsets_s": [ + 0.18977094599904376, + 0.25602599299963913, + 0.3242987109988462, + 0.39410867799961125, + 0.4754985330000636, + 0.5486001429999305, + 0.6245727040004567, + 0.7118257000001904, + 0.8034819649983547, + 0.908265993999521, + 0.9959417949976341, + 1.0664597469985893, + 1.147658178997517, + 1.2197372669979814, + 1.3063193179987138, + 1.3953999400000612, + 1.487507771998935, + 1.57448287599982, + 1.661236590000044, + 1.740908331998071, + 1.832176129999425, + 1.9189750010009448, + 2.0132027299987385, + 2.099799035000615, + 2.195891921997827, + 2.291504803000862, + 2.364673039999616, + 2.4411720099997183, + 2.512988886999665, + 2.58073460200103, + 2.656031026999699, + 2.7271688939981686, + 2.811027699000988, + 2.9145427469993592, + 3.0005804270003864, + 3.0632817980003892, + 3.1224781130003976, + 3.196271347998845, + 3.2892039350008417, + 3.385114586999407, + 3.4690845099976286, + 3.562578749999375, + 3.6616534809982113, + 3.755694288000086, + 3.8641919729998335, + 3.974287323999306, + 4.07809966700006, + 4.164769596998667, + 4.259054186000867, + 4.353502260000823, + 4.419272764000198, + 4.497193371000321, + 4.578383701998973, + 4.670473316000425, + 4.7765354689981905, + 4.887422645999322, + 5.003238732999307, + 5.10272771399832, + 5.198171723000996 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 86.77629250087193, + "p95": 110.09535099947243, + "max": 115.81608699998469 + } + }, + { + "start": 28429.66052498, + "first": 28429.755860335, + "last": 28436.210171382, + "end": 28436.210279545, + "elapsed_s": 6.549754564999603, + "chunk_offsets_s": [ + 0.09533535499940626, + 0.2558533239971439, + 0.32415824599956977, + 0.39398981599879335, + 0.47553275899917935, + 0.5483462539996253, + 0.6250674959992466, + 0.7122464450003463, + 0.8033863210002892, + 0.908530984997924, + 0.9958407149970299, + 1.066222495999682, + 1.1476100250001764, + 1.2192892659986683, + 1.3062323479971383, + 1.3953841969996574, + 1.4875543489979464, + 1.5748950239976693, + 1.661634342999605, + 1.741221147000033, + 1.8326074129981862, + 1.919453121998231, + 2.013588808997156, + 2.1003281899975264, + 2.195751481998741, + 2.2912217559969577, + 2.3647021359975042, + 2.44118835900008, + 2.512835842997447, + 2.5805599189989152, + 2.6559738969990576, + 2.7275790860003326, + 2.81095903899768, + 2.9149740779976128, + 3.000591877997067, + 3.0632651039995835, + 3.122114394998789, + 3.196189094996953, + 3.2895753829980094, + 3.3849544299991976, + 3.4690578279987676, + 3.562516464997316, + 3.661471444997005, + 3.7555462579985033, + 3.8641835859998537, + 3.9745960009968258, + 4.077855986997747, + 4.164791207997041, + 4.259290785998019, + 4.353898059998755, + 4.419657845999609, + 4.497216990999732, + 4.57855119099986, + 4.670348469997407, + 4.776356890997704, + 4.887303699997574, + 5.003666874999908, + 5.1031298529997, + 5.198787799999991, + 5.254366768000182, + 5.3077111069978855, + 5.35492000999875, + 5.409698117997323, + 5.468679646997771, + 5.526400074999401, + 5.584312784998474, + 5.6620114619981905, + 5.721230418999767, + 5.765363344999059, + 5.810237169000175, + 5.858706590999645, + 5.91854599999715, + 5.994699995997507, + 6.070744887998444, + 6.1422545469977194, + 6.216416177998326, + 6.294710853999277, + 6.360640930997761, + 6.405419787999563, + 6.45259716199871, + 6.4983488929974556, + 6.54964640199978 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 79.58680400042795, + "p95": 108.6373280013504, + "max": 160.51796899773763 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 99.9, + "per_stream_tps": 27.3, + "ttft_s_max": 0.4, + "request": { + "ttft_s": 0.238, + "decode_tps": 27.285, + "prefill_tps": 60.506, + "itl_mean_ms": 36.653, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.311, + "round_gap_p50_ms": 129.514, + "round_gap_p95_ms": 177.164, + "tokens_per_chunk": 3.633 + }, + "pooled": { + "aggregate_prefill_tps": 148.528, + "itl_mean_ms": 35.975, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.107, + "round_gap_p50_ms": 129.839, + "round_gap_p95_ms": 178.543, + "tokens_per_chunk": 3.618 + }, + "timing_valid": true, + "end_to_end_tps": 99.33829318431876, + "makespan_s": 10.308210128998326, + "steady_tps": 108.8, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 100.27783152965166, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.265747213001305, + "end_to_end_tps": 99.74919299621271, + "timing_valid": true, + "steady_tps": 108.84015690344883, + "per_stream_tps": [ + 32.0, + 25.3, + 28.2, + 26.9 + ], + "ttft_s": [ + 0.37, + 0.18, + 0.28, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.3696183400024893, + "decode_tps": 32.01788979477498, + "prefill_tps": 37.87690838042754, + "itl_mean_ms": 31.23253925882369, + "itl_p50_ms": 0.0, + "itl_p95_ms": 156.79652099788655, + "round_gap_p50_ms": 137.072874000296, + "round_gap_p95_ms": 175.2685349965759, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18478210500325076, + "decode_tps": 25.295869133100013, + "prefill_tps": 75.76491240725775, + "itl_mean_ms": 39.53214632548385, + "itl_p50_ms": 0.0, + "itl_p95_ms": 156.66141999827232, + "round_gap_p50_ms": 123.32380199950421, + "round_gap_p95_ms": 174.19039599917596, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2771181380012422, + "decode_tps": 28.19034542070741, + "prefill_tps": 50.519969934040354, + "itl_mean_ms": 35.47313752549634, + "itl_p50_ms": 0.0, + "itl_p95_ms": 157.08059099779348, + "round_gap_p50_ms": 130.37151900061872, + "round_gap_p95_ms": 175.87748399819247, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.0939329530010582, + "decode_tps": 26.917527270709325, + "prefill_tps": 149.04247713624295, + "itl_mean_ms": 37.15051497646902, + "itl_p50_ms": 0.0, + "itl_p95_ms": 156.87643499768456, + "round_gap_p50_ms": 128.85250199906295, + "round_gap_p95_ms": 176.01762599952053, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 35.84708452156823, + "itl_p50_ms": 0.0, + "itl_p95_ms": 156.79652099788655, + "round_gap_p50_ms": 129.83893699856708, + "round_gap_p95_ms": 175.73384700153838, + "round_gap_mean_ms": 131.05385739067952, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 151.4660996117577 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 28436.211868221, + "first": 28436.581486561, + "last": 28444.545784072, + "end": 28444.546294186, + "elapsed_s": 8.334425965000264, + "chunk_offsets_s": [ + 0.3696183400024893, + 0.45741819600152667, + 0.5376293650006119, + 0.6364259920010227, + 0.7463004900018859, + 0.8385060730033729, + 0.9505997790001857, + 1.0755564560022322, + 1.2020407110030646, + 1.3366920560001745, + 1.4544608650030568, + 1.573663735001901, + 1.7041527020010108, + 1.816407125999831, + 1.9379412650014274, + 2.0733694420014217, + 2.2294586440002604, + 2.374803085000167, + 2.5154767240019282, + 2.656572504001815, + 2.7887655990016356, + 2.9372379630003707, + 3.099316219002503, + 3.2615153530023235, + 3.4355618100016727, + 3.602260297000612, + 3.755951246999757, + 3.9042619690007996, + 4.027369538001949, + 4.132174461999966, + 4.249337558001571, + 4.365735747000144, + 4.506665014003374, + 4.663461535001261, + 4.8149784640008875, + 4.944349694000266, + 5.063081349002459, + 5.191814555000747, + 5.3393115650033, + 5.486795382003038, + 5.623868256003334, + 5.79913679099991, + 5.956607077001536, + 6.098001254002156, + 6.244499436001206, + 6.403732331000356, + 6.557684168001288, + 6.712261434000538, + 6.85768924200238, + 6.973900495002454, + 7.075231284001347, + 7.2027295870029775, + 7.325001951001468, + 7.452850845002104, + 7.619080811000458, + 7.805265515002247, + 8.002383344002737, + 8.175635650000913, + 8.33391585100253 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 138.87325650102866, + "p95": 175.2685349965759, + "max": 197.11782900049002 + } + }, + { + "start": 28436.211960102, + "first": 28436.396742207, + "last": 28446.47743952, + "end": 28446.47751408, + "elapsed_s": 10.265553978002572, + "chunk_offsets_s": [ + 0.18478210500325076, + 0.4576690909998433, + 0.5376071080027032, + 0.6362746250015334, + 0.7463492470014899, + 0.8379090260023077, + 0.9511968280021392, + 1.0752285100024892, + 1.2017476319997513, + 1.3364428929999121, + 1.4546114760014461, + 1.5734163400011312, + 1.7040081520026433, + 1.8159583940032462, + 1.9381194810011948, + 2.0730186930013588, + 2.2290541980000853, + 2.374293397002475, + 2.5154799000010826, + 2.656060367000464, + 2.788317544000165, + 2.9367762870024308, + 3.099293409002712, + 3.261381659001927, + 3.435572055001103, + 3.6024638910021167, + 3.7555478909998783, + 3.9042842950002523, + 4.0276080969997565, + 4.131916188001924, + 4.248707391001517, + 4.365440540001146, + 4.506657098001597, + 4.663230467001995, + 4.8145366800017655, + 4.943914101000701, + 5.063173881000694, + 5.190848662001372, + 5.339653912000358, + 5.486470735002513, + 5.623596050001652, + 5.79932989700319, + 5.9559913170014624, + 6.097491331001947, + 6.2442123260007065, + 6.403426615001081, + 6.556929124002636, + 6.7123886340014, + 6.857427242000995, + 6.974214686000778, + 7.075632936001057, + 7.202917819002323, + 7.325229031001072, + 7.452616228001716, + 7.619180175002839, + 7.8050756080010615, + 8.0014743669999, + 8.174923488000786, + 8.334691558000486, + 8.429025944002206, + 8.515111366003111, + 8.595469446001516, + 8.697995315000298, + 8.818720719002158, + 8.927924401999917, + 9.024457691000862, + 9.132221595002193, + 9.225302429000294, + 9.323200245002226, + 9.401252019000822, + 9.478864396001882, + 9.567660908000107, + 9.632596826002555, + 9.709597450000729, + 9.791707792002853, + 9.859338406000461, + 9.933659321002779, + 10.00907047200235, + 10.074175006000587, + 10.118686634999904, + 10.165244290001283, + 10.211864492001041, + 10.265479418001632 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 123.6777419999271, + "p95": 174.19039599917596, + "max": 272.88698599659256 + } + }, + { + "start": 28436.212345406, + "first": 28436.489463544, + "last": 28445.535113613, + "end": 28445.535282541, + "elapsed_s": 9.322937135002576, + "chunk_offsets_s": [ + 0.2771181380012422, + 0.45719050300249364, + 0.5373939640012395, + 0.6363445870010764, + 0.746134790002543, + 0.8377795900014462, + 0.9506738240015693, + 1.0746120680014428, + 1.201310074000503, + 1.3361254450028355, + 1.4543205250010942, + 1.573099634002574, + 1.7034711530031927, + 1.816000830000121, + 1.9372414320023381, + 2.0732556710026984, + 2.229271707001317, + 2.373674104001111, + 2.5154581520000647, + 2.655722990002687, + 2.7884458790031204, + 2.936545313001261, + 3.0989580910027144, + 3.2608560740009125, + 3.435406746000808, + 3.602120260002266, + 3.755257653003355, + 3.904049575001409, + 4.027155668001797, + 4.131979733003391, + 4.248519514003419, + 4.365471577002609, + 4.506110142003308, + 4.663190733001102, + 4.813751722002053, + 4.94359065900062, + 5.062732112000958, + 5.190807642000436, + 5.339209571000538, + 5.486007107003388, + 5.623276384001656, + 5.799153867999848, + 5.955352915003459, + 6.09719522800151, + 6.243491826000536, + 6.403177973003039, + 6.556951485999889, + 6.71170671100117, + 6.8567276040012075, + 6.9739186560000235, + 7.075031652002508, + 7.202660860002652, + 7.324066152999876, + 7.452507952002634, + 7.6184450820001075, + 7.804578625000431, + 8.000878292001289, + 8.174854061002407, + 8.334539017003408, + 8.428549887001282, + 8.51505481300046, + 8.595247750999988, + 8.69736651600033, + 8.818224925002141, + 8.928203885003313, + 9.024025334001635, + 9.131884235001053, + 9.224992537001526, + 9.32276820700281 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 131.5472040005261, + "p95": 175.87748399819247, + "max": 196.29966700085788 + } + }, + { + "start": 28436.211766867, + "first": 28436.30569982, + "last": 28445.779081139, + "end": 28445.779240831, + "elapsed_s": 9.567473964001692, + "chunk_offsets_s": [ + 0.0939329530010582, + 0.45736939500056906, + 0.5375413779984228, + 0.6364029260003008, + 0.7463160340012109, + 0.838860277999629, + 0.9504469710009289, + 1.0750639760008198, + 1.2020744930014189, + 1.3365142190014012, + 1.4543582890000835, + 1.5734827390006103, + 1.7044777320006688, + 1.816375035999954, + 1.9376095490006264, + 2.0729499439985375, + 2.228960033000476, + 2.374153568998736, + 2.516083366001112, + 2.656045532999997, + 2.7889716450008564, + 2.9368801150012587, + 3.0992746470001293, + 3.261291576000076, + 3.435346856000251, + 3.602227622999635, + 3.7556149160009227, + 3.9042824970019865, + 4.027398469999753, + 4.131961249000597, + 4.248793651000597, + 4.365543323001475, + 4.50705324700175, + 4.663929681999434, + 4.814881785001489, + 4.943734287000552, + 5.0635101279985975, + 5.191688382001303, + 5.339696658000321, + 5.48698193400196, + 5.623636717999034, + 5.799654343998554, + 5.956486651000887, + 6.097883385999012, + 6.243939883999701, + 6.403689047001535, + 6.557478369999444, + 6.7126337090012385, + 6.857093136000913, + 6.974057601000823, + 7.075381250000646, + 7.2028811759992095, + 7.324840541001322, + 7.453311933000805, + 7.618887420001556, + 7.80541715399886, + 8.002025824000157, + 8.17538155100192, + 8.33427315299923, + 8.429024624001613, + 8.515116140999453, + 8.595553336999728, + 8.698075798998616, + 8.818565891000617, + 8.927755259999685, + 9.024306175000675, + 9.132141144000343, + 9.225050870001724, + 9.323205078999308, + 9.401655964000383, + 9.47891793799863, + 9.567314272000658 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 128.85250199906295, + "p95": 176.01762599952053, + "max": 363.43644199951086 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 99.88471873737402, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.308210128998326, + "end_to_end_tps": 99.33829318431876, + "timing_valid": true, + "steady_tps": 108.23164156802089, + "per_stream_tps": [ + 30.8, + 25.7, + 28.1, + 27.0 + ], + "ttft_s": [ + 0.1, + 0.38, + 0.28, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09593322500222712, + "decode_tps": 30.763275270127128, + "prefill_tps": 145.9348416533999, + "itl_mean_ms": 32.50629171371282, + "itl_p50_ms": 0.0, + "itl_p95_ms": 160.03981600078987, + "round_gap_p50_ms": 138.09547599885263, + "round_gap_p95_ms": 182.16177700014669, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.37575451500015333, + "decode_tps": 25.676874241821725, + "prefill_tps": 37.25836800655419, + "itl_mean_ms": 38.94555040392066, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.54857299948344, + "round_gap_p50_ms": 123.77985900093336, + "round_gap_p95_ms": 175.5229340014921, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28202311899804045, + "decode_tps": 28.057169747666897, + "prefill_tps": 49.64132036316241, + "itl_mean_ms": 35.64151370197114, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.7634610006935, + "round_gap_p50_ms": 129.1930709994631, + "round_gap_p95_ms": 178.54262300170376, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19026604899772792, + "decode_tps": 27.025562433449643, + "prefill_tps": 73.58117790193448, + "itl_mean_ms": 37.00200513726575, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.88389300036943, + "round_gap_p50_ms": 128.80270399909932, + "round_gap_p95_ms": 178.57117599851335, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 36.02384023921759, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.68398000040906, + "round_gap_p50_ms": 128.80270399909932, + "round_gap_p95_ms": 178.54262300170376, + "round_gap_mean_ms": 131.70006108961272, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 148.52849544036923 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 28446.480795098, + "first": 28446.576728323, + "last": 28454.86583271, + "end": 28454.866015675, + "elapsed_s": 8.385220577001746, + "chunk_offsets_s": [ + 0.09593322500222712, + 0.4659209740020742, + 0.5515467610021005, + 0.6525862830021651, + 0.7593564950002474, + 0.8485519179994299, + 0.9612500530020043, + 1.086838393999642, + 1.2156978780003556, + 1.3497642680013087, + 1.4709563010001148, + 1.5963512839989562, + 1.7148780210009136, + 1.840494125000987, + 1.9629937460013025, + 2.101089222000155, + 2.25924306700108, + 2.405801838001935, + 2.557603018001828, + 2.68607501400038, + 2.8145559430013236, + 2.9615749629992933, + 3.127624533000926, + 3.2895207239998854, + 3.4683354500011774, + 3.645677524000348, + 3.7949884549998387, + 3.9413824799994472, + 4.0654286880017025, + 4.1724614779996045, + 4.289097979999497, + 4.408242964000237, + 4.558481777999987, + 4.7185215940007765, + 4.8620137960024294, + 4.993120852999709, + 5.114991867001663, + 5.241984929001774, + 5.3917263809998985, + 5.545491261000279, + 5.68350991600164, + 5.8453009400000155, + 6.001057908000803, + 6.143637663000845, + 6.291240799000661, + 6.454918371000531, + 6.616404824999336, + 6.760381754000264, + 6.9013547399990784, + 7.021621583000524, + 7.124045307002234, + 7.251043033000315, + 7.372351579000679, + 7.505606219001493, + 7.674142358999234, + 7.85630413599938, + 8.051014700999076, + 8.226019910001924, + 8.385037611998996 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 139.53423099883366, + "p95": 182.16177700014669, + "max": 369.9877489998471 + } + }, + { + "start": 28446.481630715, + "first": 28446.85738523, + "last": 28456.788500583, + "end": 28456.78856333, + "elapsed_s": 10.306932614999823, + "chunk_offsets_s": [ + 0.37575451500015333, + 0.46503734799989616, + 0.5504206200021144, + 0.6522778549988288, + 0.7588099360000342, + 0.8480306860001292, + 0.960256415000913, + 1.086493833001441, + 1.215104250000877, + 1.3486529959991458, + 1.4704201620006643, + 1.5957591280021006, + 1.7139750490023289, + 1.8402042699999583, + 1.962226872001338, + 2.100527474001865, + 2.258321708999574, + 2.4047959060007997, + 2.555813990002207, + 2.6853014799999073, + 2.813904184000421, + 2.960810124001, + 3.1271614340002998, + 3.2888841989988578, + 3.467080333000922, + 3.6445868560003873, + 3.794659988001513, + 3.941139046000899, + 4.064918905001832, + 4.171541784999135, + 4.288744348999899, + 4.407469148001837, + 4.557870941000147, + 4.717939917001786, + 4.861852820999047, + 4.992042100999242, + 5.114574269999139, + 5.241784945999825, + 5.391260479998891, + 5.544967391000682, + 5.683092883002246, + 5.845068175000051, + 6.000093565002317, + 6.143614405002154, + 6.290348842001549, + 6.454992964001576, + 6.615907162002259, + 6.759743616999913, + 6.9005852789996425, + 7.0206478879990755, + 7.123465614000452, + 7.250840097000037, + 7.37159629800226, + 7.504472386000998, + 7.673608567001793, + 7.855959922999318, + 8.050026622000587, + 8.225549556002079, + 8.385098129001562, + 8.481450431001576, + 8.57451430600122, + 8.655444529998931, + 8.757147086998884, + 8.872823739999149, + 8.98178763100077, + 9.079762096000195, + 9.190339347998815, + 9.28236316499897, + 9.36997175400029, + 9.44847448300061, + 9.532860758001334, + 9.625096457999462, + 9.690352322999388, + 9.767280668002059, + 9.835779049000848, + 9.903269016002014, + 9.975474417999067, + 10.052323505999084, + 10.115461942001275, + 10.161695730999782, + 10.208066146999045, + 10.255062252999778, + 10.306869867999922 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 124.55941250118485, + "p95": 175.5229340014921, + "max": 194.06669900126872 + } + }, + { + "start": 28446.481215232, + "first": 28446.763238351, + "last": 28455.851824345, + "end": 28455.852113785, + "elapsed_s": 9.370898552999279, + "chunk_offsets_s": [ + 0.28202311899804045, + 0.4651009730005171, + 0.5512936220002302, + 0.6519018920007511, + 0.7594509749978897, + 0.847862618000363, + 0.9603806560007797, + 1.0866093879994878, + 1.215349485999468, + 1.3485218089990667, + 1.4709670709999045, + 1.5962801559981017, + 1.7148631339987332, + 1.8405168379977113, + 1.9626857269977336, + 2.101118504000624, + 2.2584665440008393, + 2.4057594219993916, + 2.556929221998871, + 2.686122292998334, + 2.81468403299732, + 2.9610803989999113, + 3.127516492000723, + 3.2891683839989128, + 3.4677110070006165, + 3.6451336779973644, + 3.795395289998851, + 3.9418726730000344, + 4.065736320000724, + 4.171539348000806, + 4.28905352899892, + 4.407535817997996, + 4.558347299000161, + 4.717984432998492, + 4.861765593999735, + 4.992643370998849, + 5.1151746139985335, + 5.241824193999491, + 5.391441293999378, + 5.545000002999586, + 5.683458523999434, + 5.845809087000816, + 6.001211089998833, + 6.1436726249994535, + 6.2906263080003555, + 6.455179844997474, + 6.616426134998619, + 6.760700090999308, + 6.900682082999992, + 7.021394186998805, + 7.124059702997329, + 7.251711172000796, + 7.37215556999945, + 7.505419664998044, + 7.67423959900043, + 7.856166405999829, + 8.05095233999964, + 8.225523817000067, + 8.38528727800076, + 8.481664515999, + 8.574626014000387, + 8.6554556029987, + 8.757626274000359, + 8.87302180199913, + 8.982013057000586, + 9.0803112959984, + 9.190362911998818, + 9.282456072000059, + 9.370609113000683 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 130.03542399928847, + "p95": 178.54262300170376, + "max": 194.78593399981037 + } + }, + { + "start": 28446.480353201, + "first": 28446.67061925, + "last": 28456.10613056, + "end": 28456.10624848, + "elapsed_s": 9.625895278997632, + "chunk_offsets_s": [ + 0.19026604899772792, + 0.4658622839997406, + 0.5517646640000748, + 0.6533091079982114, + 0.760165503997996, + 0.8492445390002104, + 0.9610156809976615, + 1.0875383159982448, + 1.216268547999789, + 1.349439027999324, + 1.4714841620007064, + 1.5966640359984012, + 1.7155289359980088, + 1.8408226949977688, + 1.9635685520006518, + 2.101902288999554, + 2.259281231999921, + 2.406465047999518, + 2.5577090859987948, + 2.686645194000448, + 2.8154478979995474, + 2.9623009899987665, + 3.1283165349996125, + 3.2903020230005495, + 3.468873198999063, + 3.6461579559982056, + 3.7956876370008104, + 3.9419641729982686, + 4.066504159000033, + 4.172238221999578, + 4.2898059859981, + 4.408360754998284, + 4.559039058000053, + 4.718723038000462, + 4.862569542998244, + 4.993409236998559, + 5.115576726999279, + 5.2426024619999225, + 5.392085355997551, + 5.546002923998458, + 5.684089685997606, + 5.845882082998287, + 6.001881828997284, + 6.144249570999818, + 6.291535546999512, + 6.455437810000149, + 6.6170620230004715, + 6.76130469599957, + 6.901490092997847, + 7.021983127997373, + 7.125287503997242, + 7.251803753999411, + 7.373074937997444, + 7.506179702999361, + 7.6749929129982775, + 7.857087435000722, + 8.051191267000831, + 8.226315142997919, + 8.386199035998288, + 8.482351664999442, + 8.575281502999132, + 8.656252795997716, + 8.758244017000834, + 8.873835304999375, + 8.982668131000537, + 9.081121699997311, + 9.191030572997988, + 9.283126958998764, + 9.371560083000077, + 9.44982069499747, + 9.534259882999322, + 9.625777359000494 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 128.80270399909932, + "p95": 178.57117599851335, + "max": 275.5962350020127 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 99.87287152809446, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.312685895998584, + "end_to_end_tps": 99.29517977439043, + "timing_valid": true, + "steady_tps": 110.6715319808621, + "per_stream_tps": [ + 31.5, + 25.7, + 27.5, + 27.1 + ], + "ttft_s": [ + 0.3, + 0.4, + 0.1, + 0.2 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.3018111179990228, + "decode_tps": 31.531212580038563, + "prefill_tps": 46.38662781152193, + "itl_mean_ms": 31.714606517640526, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.8730780024198, + "round_gap_p50_ms": 137.63282899890328, + "round_gap_p95_ms": 181.69562999901245, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.39876802700018743, + "decode_tps": 25.721943738707704, + "prefill_tps": 35.10813067265651, + "itl_mean_ms": 38.877310756851884, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.09928199835122, + "round_gap_p50_ms": 123.17215700022643, + "round_gap_p95_ms": 173.57780399834155, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09954769699834287, + "decode_tps": 27.50229662862053, + "prefill_tps": 140.63610130762797, + "itl_mean_ms": 36.36059975294355, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.18448999946122, + "round_gap_p50_ms": 134.77049300126964, + "round_gap_p95_ms": 178.31032700269134, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19860599899766385, + "decode_tps": 27.066782183098486, + "prefill_tps": 70.49132488774762, + "itl_mean_ms": 36.945655129424196, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.4368090009084, + "round_gap_p50_ms": 129.8348950003856, + "round_gap_p95_ms": 178.759247999551, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 35.97454303921504, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.10743399945204, + "round_gap_p50_ms": 130.71238400152652, + "round_gap_p95_ms": 178.759247999551, + "round_gap_mean_ms": 131.51983476702273, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 140.38758344411042 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 28456.790495115, + "first": 28457.092306233, + "last": 28465.179530895, + "end": 28465.179727426, + "elapsed_s": 8.389232310997613, + "chunk_offsets_s": [ + 0.3018111179990228, + 0.4859331320003548, + 0.5671280299975479, + 0.6675318979978329, + 0.7760911389996181, + 0.8655921479985409, + 0.978094037000119, + 1.1037392309990537, + 1.2396463400000357, + 1.376274474998354, + 1.495618580000155, + 1.616137269000319, + 1.736661932998686, + 1.8417252299987013, + 1.9649369869985094, + 2.103392220000387, + 2.2675180829974124, + 2.4167459689997486, + 2.5603121999993164, + 2.688224679997802, + 2.8189370639993285, + 2.9655043249986193, + 3.131281592999585, + 3.304972858997644, + 3.4789333369990345, + 3.650234933997126, + 3.802004762997967, + 3.947887460999482, + 4.072472298998036, + 4.181694685998082, + 4.3048560219976935, + 4.424566458998015, + 4.567899794998084, + 4.728157673998794, + 4.872659861997818, + 5.002415427999949, + 5.124570746997051, + 5.259700943999633, + 5.4086964879970765, + 5.556682042999455, + 5.693856226000207, + 5.852963659999659, + 6.00791596399722, + 6.148962857998413, + 6.303210334997857, + 6.463083413000277, + 6.616156830998079, + 6.759897310999804, + 6.8975301399987075, + 7.015471744998649, + 7.117369117997441, + 7.2520904049997625, + 7.375928977999138, + 7.504832776998228, + 7.67271740599972, + 7.854413035998732, + 8.048199140997895, + 8.22738655899957, + 8.389035779997357 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 138.04403100039053, + "p95": 181.69562999901245, + "max": 193.7861049991625 + } + }, + { + "start": 28456.79014267, + "first": 28457.188910697, + "last": 28467.10262494, + "end": 28467.102700917, + "elapsed_s": 10.312558246998378, + "chunk_offsets_s": [ + 0.39876802700018743, + 0.4861161769986211, + 0.5675463849984226, + 0.6682653569987451, + 0.7761905699990166, + 0.8656949029973475, + 0.978504801998497, + 1.1037945949974528, + 1.2397404539988202, + 1.376690408000286, + 1.4953976359975059, + 1.6164251230002264, + 1.7366378769984294, + 1.8421635189988592, + 1.9648782379990735, + 2.103669264997734, + 2.2681878539988247, + 2.4165235010004835, + 2.5608543599992117, + 2.6893254530004924, + 2.8182290980003017, + 2.9662162069980695, + 3.1316850229995907, + 3.3052628269979323, + 3.4798503710007935, + 3.650812911000685, + 3.8022815769982117, + 3.9489682690000336, + 4.072766977998981, + 4.181991682999069, + 4.305163839999295, + 4.424477205997391, + 4.568194441999367, + 4.728297229998134, + 4.87348320700039, + 5.003461009997409, + 5.124824916998477, + 5.260006345000875, + 5.409125771999243, + 5.55711757699828, + 5.694158541999059, + 5.85325782399741, + 6.008569149998948, + 6.1490222360007465, + 6.304039219998231, + 6.463163892000011, + 6.61623689499902, + 6.76050173999829, + 6.897754298999644, + 7.015989585997886, + 7.117900851997547, + 7.252028814000369, + 7.37634763999813, + 7.504787380999915, + 7.672892047998175, + 7.854365193998092, + 8.048612304999551, + 8.22763249700074, + 8.38902445200074, + 8.482465327997488, + 8.57116789399879, + 8.65171027299948, + 8.755932345000474, + 8.87241037000058, + 8.979764004998287, + 9.077582162997714, + 9.189656801998353, + 9.286652842998592, + 9.37200631000087, + 9.449632813997596, + 9.529071360997477, + 9.619502801000635, + 9.68484822499886, + 9.76191899100013, + 9.828821180999512, + 9.896656516000803, + 9.970486390997394, + 10.046440123998764, + 10.112872057998175, + 10.1592273440001, + 10.210935322000296, + 10.259871127000224, + 10.312482269997417 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 123.48543299958692, + "p95": 173.57780399834155, + "max": 194.24711100145942 + } + }, + { + "start": 28456.790093633, + "first": 28456.88964133, + "last": 28466.161594267, + "end": 28466.161734736, + "elapsed_s": 9.371641103000002, + "chunk_offsets_s": [ + 0.09954769699834287, + 0.4859867060004035, + 0.5673358569983975, + 0.6677620619993831, + 0.7761285919987131, + 0.8664622680007597, + 0.9778365659985866, + 1.1039418119980837, + 1.2396466789978149, + 1.3765996529982658, + 1.4952583509984834, + 1.616295904997969, + 1.7364396099983423, + 1.8419915859994944, + 1.9646834859995579, + 2.10354857999846, + 2.268173715998273, + 2.4162283309997292, + 2.5601809910003794, + 2.6890694059984526, + 2.818084327998804, + 2.965797979999479, + 3.131517541998619, + 3.3046435639989795, + 3.479545166999742, + 3.6513276509977004, + 3.8018356940010563, + 3.948062420000497, + 4.0732899850008835, + 4.182618265000201, + 4.305924797998159, + 4.4247273199980555, + 4.568029168000066, + 4.727846931000386, + 4.873174877000565, + 5.003450054999121, + 5.124681400000554, + 5.259672385000158, + 5.409779597001034, + 5.556837069998437, + 5.693823222998617, + 5.852928626998619, + 6.008017760999792, + 6.149175579997973, + 6.303798467000888, + 6.462982957000349, + 6.61609894600042, + 6.760897971998929, + 6.897662389998004, + 7.015515479997703, + 7.117877677999786, + 7.252648171001056, + 7.376602119999006, + 7.505069406000985, + 7.673543171000347, + 7.854688976000034, + 8.049297425997793, + 8.227607753000484, + 8.388861577997886, + 8.482320071998402, + 8.570721389998653, + 8.651596192998113, + 8.755899911000597, + 8.872281021998788, + 8.97967062799944, + 9.077445612998417, + 9.189573315998132, + 9.286869001000014, + 9.371500633998949 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 134.88073900043673, + "p95": 178.31032700269134, + "max": 386.43900900206063 + } + }, + { + "start": 28456.790015021, + "first": 28456.98862102, + "last": 28466.409763078, + "end": 28466.409875888, + "elapsed_s": 9.619860866998351, + "chunk_offsets_s": [ + 0.19860599899766385, + 0.4859530559988343, + 0.5675031799983117, + 0.6681551080000645, + 0.775955499997508, + 0.8662586270002066, + 0.9781998600010411, + 1.1044073279990698, + 1.239813772001071, + 1.3770384829986142, + 1.4957035400002496, + 1.616492303997802, + 1.7366548019999755, + 1.8421442010003375, + 1.9649429579985735, + 2.103734985998017, + 2.267636214997765, + 2.4169097279991547, + 2.5607069219986442, + 2.6888864030006516, + 2.818721298001037, + 2.965705351998622, + 3.131706194999424, + 3.305135695998615, + 3.47984593900037, + 3.6508019419998163, + 3.8022580459983146, + 3.948717528000998, + 4.073164386998542, + 4.182369191999896, + 4.305908353999257, + 4.424875477998285, + 4.568549797997548, + 4.728516753999429, + 4.873509486998955, + 5.003230725000321, + 5.125177760997758, + 5.259969525999622, + 5.409508235999965, + 5.557621637999546, + 5.69419050599754, + 5.853294295000524, + 6.0085993649991, + 6.149468736999552, + 6.3037988449978, + 6.463235653998709, + 6.616298074997758, + 6.760454936000315, + 6.89793785099755, + 7.01573429400014, + 7.118105763998756, + 7.252033542998106, + 7.3761559249978745, + 7.504794475000381, + 7.672944687998097, + 7.854544590998557, + 8.049057300999266, + 8.227816548998817, + 8.389221933000954, + 8.482537437997962, + 8.571235701998376, + 8.651891751000221, + 8.755618746999971, + 8.872630170000775, + 8.979843548000645, + 9.077667892001045, + 9.189892927999608, + 9.286579845000233, + 9.372094548998575, + 9.449825980998867, + 9.52936234200024, + 9.619748057000834 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 129.8348950003856, + "p95": 178.759247999551, + "max": 287.34705700117047 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 107.2, + "per_stream_tps": 23.7, + "ttft_s_max": 0.49, + "request": { + "ttft_s": 0.286, + "decode_tps": 23.698, + "prefill_tps": 48.866, + "itl_mean_ms": 42.198, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.122, + "round_gap_p50_ms": 158.104, + "round_gap_p95_ms": 196.256, + "tokens_per_chunk": 3.71 + }, + "pooled": { + "aggregate_prefill_tps": 145.015, + "itl_mean_ms": 41.976, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.959, + "round_gap_p50_ms": 158.369, + "round_gap_p95_ms": 197.262, + "tokens_per_chunk": 3.657 + }, + "timing_valid": true, + "end_to_end_tps": 106.773219497359, + "makespan_s": 11.988024768998002, + "steady_tps": 118.7, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 107.0117178347154, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 12.011363001998689, + "end_to_end_tps": 106.56575775680147, + "timing_valid": true, + "steady_tps": 116.4167460780569, + "per_stream_tps": [ + 26.5, + 21.9, + 23.2, + 23.5, + 24.1 + ], + "ttft_s": [ + 0.19, + 0.38, + 0.1, + 0.49, + 0.29 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19076182899880223, + "decode_tps": 26.515515125895828, + "prefill_tps": 73.38994427489948, + "itl_mean_ms": 37.71376853332828, + "itl_p50_ms": 0.0, + "itl_p95_ms": 183.39422899953206, + "round_gap_p50_ms": 163.44017799929134, + "round_gap_p95_ms": 199.1724260005867, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38154591499915114, + "decode_tps": 21.927364561147954, + "prefill_tps": 36.692831582356604, + "itl_mean_ms": 45.60511580000143, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.0376889995241, + "round_gap_p50_ms": 151.1450899997726, + "round_gap_p95_ms": 195.32371500099543, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09635810699910508, + "decode_tps": 23.240060162087172, + "prefill_tps": 145.29135571467822, + "itl_mean_ms": 43.02914850587852, + "itl_p50_ms": 0.0, + "itl_p95_ms": 183.78311899869004, + "round_gap_p50_ms": 161.09855100148707, + "round_gap_p95_ms": 197.41570900077932, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.486050883999269, + "decode_tps": 23.545855544945525, + "prefill_tps": 28.803568640399913, + "itl_mean_ms": 42.47031916471029, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.20366699824808, + "round_gap_p50_ms": 156.8510460019752, + "round_gap_p95_ms": 195.30901999678463, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2872416069985775, + "decode_tps": 24.051920930380945, + "prefill_tps": 48.739457163910565, + "itl_mean_ms": 41.57672074902175, + "itl_p50_ms": 0.0, + "itl_p95_ms": 183.72096399980364, + "round_gap_p50_ms": 162.2563929995522, + "round_gap_p95_ms": 197.26184400133207, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 42.079014550588056, + "itl_p50_ms": 0.0, + "itl_p95_ms": 183.39422899953206, + "round_gap_p50_ms": 161.09855100148707, + "round_gap_p95_ms": 197.26184400133207, + "round_gap_mean_ms": 155.50940159999934, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 143.96916894476777 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 28467.105341137, + "first": 28467.296102966, + "last": 28476.913113942, + "end": 28476.913691314, + "elapsed_s": 9.80835017699792, + "chunk_offsets_s": [ + 0.19076182899880223, + 0.584341804998985, + 0.6744872239978577, + 0.7869377940005506, + 0.9095034709971515, + 1.0146238909983367, + 1.1469137019994378, + 1.2947866969989263, + 1.4471133570004895, + 1.5991172189969802, + 1.7247731139977986, + 1.8645385130002978, + 2.016125865000504, + 2.1519775229971856, + 2.2919533719978062, + 2.4489782359996752, + 2.6210927829997672, + 2.793249061000097, + 2.9701760539974202, + 3.132057036997139, + 3.2953167669984396, + 3.477019278998341, + 3.676191704998928, + 3.863599524000165, + 4.05867647299965, + 4.24675277300048, + 4.417350542000349, + 4.587207373999263, + 4.741198065999924, + 4.883421556998655, + 5.0457971329997235, + 5.202473588997236, + 5.381461002998549, + 5.566148145997431, + 5.744014636999054, + 5.907454814998346, + 6.054176383997401, + 6.203492675998859, + 6.384130566999374, + 6.565476765998028, + 6.728176770997379, + 6.902056644998083, + 7.075508515998081, + 7.241134939999029, + 7.424529168998561, + 7.611839758999849, + 7.786724710000271, + 7.952433088998077, + 8.116438884997478, + 8.278651048000029, + 8.40197055200042, + 8.539284181999392, + 8.676313174997631, + 8.834294324999064, + 9.028639909000049, + 9.22643097299806, + 9.434350960997108, + 9.622118256000249, + 9.807772804997512 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 163.72298699934618, + "p95": 199.1724260005867, + "max": 393.5799760001828 + } + }, + { + "start": 28467.105667059, + "first": 28467.487212974, + "last": 28479.116517503, + "end": 28479.116624181, + "elapsed_s": 12.010957121998217, + "chunk_offsets_s": [ + 0.38154591499915114, + 0.5832234529989364, + 0.6739834309992148, + 0.7864332450008078, + 0.9090060210000956, + 1.0141009549988667, + 1.1461431469979289, + 1.2942989519979164, + 1.446897245998116, + 1.59825495199766, + 1.7243699799983006, + 1.8640812240009836, + 2.0152263140007562, + 2.1512609210003575, + 2.2911859240011836, + 2.4484237799988477, + 2.620302151000942, + 2.792441773999599, + 2.9704929439976695, + 3.1313268989979406, + 3.2944050199985213, + 3.475891704998503, + 3.675735868000629, + 3.8634060459989996, + 4.0575865549981245, + 4.245832443997642, + 4.41709518299831, + 4.586947394000163, + 4.7412328169994, + 4.883298373999423, + 5.045794083998771, + 5.202564609000547, + 5.380948124999122, + 5.565548984999623, + 5.743723701998533, + 5.9067597719986225, + 6.054587680999248, + 6.204185243997927, + 6.383712316001038, + 6.566046288000507, + 6.72799815399776, + 6.902124490999995, + 7.0753962119997595, + 7.240602276000573, + 7.424639965000097, + 7.612161967001157, + 7.786494173000392, + 7.952195035999466, + 8.116532464999182, + 8.27861750000011, + 8.400964624001062, + 8.539150259999587, + 8.676319013000466, + 8.833318740998948, + 9.028642455999943, + 9.225811505999445, + 9.433966736000002, + 9.621687161998125, + 9.807350921000761, + 9.945049437999842, + 10.07391415299935, + 10.192239637999592, + 10.335916324998834, + 10.486451952998323, + 10.61596502500106, + 10.740989932000957, + 10.888950754000689, + 10.984422867997637, + 11.068514489001245, + 11.145787127999938, + 11.224539061000542, + 11.315675730998919, + 11.380789061000542, + 11.45726051100064, + 11.525623147997976, + 11.592809426998429, + 11.66689573199983, + 11.743834182998398, + 11.80921214800037, + 11.854742794999765, + 11.905699277998792, + 11.957403896998585, + 12.010850443999516 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 151.25139799965837, + "p95": 195.32371500099543, + "max": 208.1552300005569 + } + }, + { + "start": 28467.105576341, + "first": 28467.201934448, + "last": 28478.174367317, + "end": 28478.174604498, + "elapsed_s": 11.069028156998684, + "chunk_offsets_s": [ + 0.09635810699910508, + 0.5838623630006623, + 0.6740226670008269, + 0.7871215449995361, + 0.9090244850012823, + 1.0146554919992923, + 1.1464415919981548, + 1.2944854749985097, + 1.4466764380013046, + 1.5985835200008296, + 1.7240691389997664, + 1.863743894999061, + 2.015713644999778, + 2.1517989999993006, + 2.2916741970002477, + 2.448320853000041, + 2.62079389399878, + 2.7926159409980755, + 2.970028823998291, + 3.131127374999778, + 3.294753154001228, + 3.47609818699857, + 3.67513188799785, + 3.8632487619979656, + 4.058162661000097, + 4.246371420998912, + 4.416966552998929, + 4.586911247999524, + 4.741632495999511, + 4.882590769000672, + 5.045933747998788, + 5.202120265999838, + 5.3815078970001196, + 5.566027191998728, + 5.743329559998529, + 5.906559656999889, + 6.053851936998399, + 6.203077996000502, + 6.383208400999138, + 6.56605866999962, + 6.727770707999298, + 6.901734244998806, + 7.075121001998923, + 7.240460555000027, + 7.424243673998717, + 7.61151852599869, + 7.786339336998935, + 7.952455639999243, + 8.116394067001238, + 8.279062173998682, + 8.401384384000266, + 8.538955143998464, + 8.676004189997911, + 8.833616845000506, + 9.028220762000274, + 9.225636471001053, + 9.433980490000977, + 9.621602444000018, + 9.808067591999134, + 9.94525700200029, + 10.074355778000609, + 10.192770030000247, + 10.336370999000792, + 10.486932724001235, + 10.616253711999889, + 10.741971964998811, + 10.889124977999018, + 10.984387575001165, + 11.068790975998127 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 161.4052945005824, + "p95": 197.41570900077932, + "max": 487.5042560015572 + } + }, + { + "start": 28467.105425507, + "first": 28467.591476391, + "last": 28478.421407778, + "end": 28478.421571512, + "elapsed_s": 11.31614600499961, + "chunk_offsets_s": [ + 0.486050883999269, + 0.5843404789993656, + 0.6740139369976532, + 0.7867298629971629, + 0.9093130309993285, + 1.0144099479985016, + 1.1465030649997061, + 1.294900011998834, + 1.4473319419994368, + 1.5986073229978501, + 1.7243158119999862, + 1.8640866469977482, + 2.015528848998656, + 2.151774800997373, + 2.291526848999638, + 2.449229590998584, + 2.6210724499978824, + 2.79320652000024, + 2.970485906000249, + 3.1317835899972124, + 3.2944372849997308, + 3.476557267997123, + 3.6759342000004835, + 3.863435428997036, + 4.057923054999264, + 4.24614236799971, + 4.417525559998467, + 4.5867289019988675, + 4.741210803997092, + 4.882859227000154, + 5.0461351109988755, + 5.202878915999463, + 5.380786756999441, + 5.565721123999538, + 5.743627233998268, + 5.906876454999292, + 6.054276333998132, + 6.204138793000311, + 6.383481816999847, + 6.565685483998095, + 6.728325206997397, + 6.902403124997363, + 7.075597259998176, + 7.24078757299867, + 7.424808616000519, + 7.612167706000037, + 7.786834787999396, + 7.952673506999417, + 8.116850307997083, + 8.278920120999828, + 8.4017243399976, + 8.539482992000558, + 8.676788289998512, + 8.833639336000488, + 9.028948355997272, + 9.22651023799699, + 9.434341620999476, + 9.621852996999223, + 9.808864799997536, + 9.945570944997598, + 10.074237067998183, + 10.192527307997807, + 10.336107953000464, + 10.487329835999844, + 10.617042268997466, + 10.741342366000026, + 10.889094103997195, + 10.984622554999078, + 11.069054084000527, + 11.145952513998054, + 11.225039829998423, + 11.315982271000394 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 156.8510460019752, + "p95": 195.30901999678463, + "max": 207.83138300248538 + } + }, + { + "start": 28467.105261179, + "first": 28467.392502786, + "last": 28477.994566577, + "end": 28477.994848773, + "elapsed_s": 10.88958759399975, + "chunk_offsets_s": [ + 0.2872416069985775, + 0.5835437150017242, + 0.6742361590004293, + 0.7866020270012086, + 0.9091186660007224, + 1.0151250419985445, + 1.1466060269995069, + 1.2945104719983647, + 1.447556388000521, + 1.5987270260011428, + 1.724715276999632, + 1.8641678630010574, + 2.0159673759990255, + 2.152304881998134, + 2.291733980000572, + 2.449119385000813, + 2.6207922940011485, + 2.792781552998349, + 2.9700561739991826, + 3.1312715280000702, + 3.294855878000817, + 3.476534902001731, + 3.6755485360008606, + 3.8637510369990196, + 4.057914659999369, + 4.246150123999541, + 4.417730006000056, + 4.586955766000756, + 4.741126198001439, + 4.883270425998489, + 5.046000310998352, + 5.20279107999886, + 5.3808819920013775, + 5.565755483999965, + 5.74372667300122, + 5.906948854000802, + 6.05451265100055, + 6.2041086470017035, + 6.383884720999049, + 6.566023562998453, + 6.728450694001367, + 6.9023589360003825, + 7.07568005199937, + 7.241139375000785, + 7.424860339000588, + 7.61228957899948, + 7.786951622998458, + 7.952138847998867, + 8.116894274000515, + 8.279150667000067, + 8.401958004000335, + 8.53961073300161, + 8.676783147999231, + 8.834032030001254, + 9.029181140998844, + 9.226442985000176, + 9.434767144000944, + 9.621694916000706, + 9.808904637000524, + 9.945664225000655, + 10.074513332998322, + 10.192586750999908, + 10.336361894998845, + 10.487000152999826, + 10.617368890001671, + 10.742088005001278, + 10.889305397999124 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 162.3417620012333, + "p95": 197.26184400133207, + "max": 296.3021080031467 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 107.58375544291012, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 11.949115551000432, + "end_to_end_tps": 107.12089899346842, + "timing_valid": true, + "steady_tps": 119.01183189909409, + "per_stream_tps": [ + 26.6, + 21.9, + 24.0, + 22.8, + 24.6 + ], + "ttft_s": [ + 0.19, + 0.29, + 0.38, + 0.1, + 0.48 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19196187099805684, + "decode_tps": 26.648950204900313, + "prefill_tps": 72.93114995811703, + "itl_mean_ms": 37.524930337260194, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.553555001301, + "round_gap_p50_ms": 163.21521799909533, + "round_gap_p95_ms": 200.4434129994479, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28587423199860496, + "decode_tps": 21.864865617637175, + "prefill_tps": 48.972584559731565, + "itl_mean_ms": 45.735474321568915, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.1216300013475, + "round_gap_p50_ms": 149.5297120018222, + "round_gap_p95_ms": 193.50805599970045, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3800479709971114, + "decode_tps": 23.982757525934336, + "prefill_tps": 36.837454922516635, + "itl_mean_ms": 41.69662303922415, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.29225799950655, + "round_gap_p50_ms": 159.29042900097556, + "round_gap_p95_ms": 196.2520199995197, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09748453800057177, + "decode_tps": 22.84866891484802, + "prefill_tps": 143.61251832488438, + "itl_mean_ms": 43.7662256705973, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.9916029999149, + "round_gap_p50_ms": 156.76645200073835, + "round_gap_p95_ms": 196.25569899653783, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.48270804899948416, + "decode_tps": 24.64447519834292, + "prefill_tps": 29.003038231945787, + "itl_mean_ms": 40.57704584706431, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.6002879968437, + "round_gap_p50_ms": 158.560594998562, + "round_gap_p95_ms": 194.2819360010617, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 41.86005984314297, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.959426998219, + "round_gap_p50_ms": 158.36903499803157, + "round_gap_p95_ms": 196.25569899653783, + "round_gap_mean_ms": 154.7002211594414, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 145.01519115972894 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 28479.119257302, + "first": 28479.311219173, + "last": 28488.880076409, + "end": 28488.880234392, + "elapsed_s": 9.76097708999805, + "chunk_offsets_s": [ + 0.19196187099805684, + 0.5783568059996469, + 0.6696346389981045, + 0.7845029969976167, + 0.9071986929993727, + 1.0118516449983872, + 1.1451260699977865, + 1.2914895159992739, + 1.4456449749995954, + 1.597156357998756, + 1.7237424249979085, + 1.862300659999164, + 2.0145907729965984, + 2.14996021999832, + 2.2895416189967364, + 2.448793443996692, + 2.619396327998402, + 2.788283992998913, + 2.9658207609973033, + 3.1286368029977893, + 3.2926597249970655, + 3.473946916998102, + 3.67439032999755, + 3.863540842998191, + 4.057716498999071, + 4.244918151998718, + 4.414238592999027, + 4.5812604049970105, + 4.731263120997028, + 4.873944902999938, + 5.037160120999033, + 5.193895661999704, + 5.369168274999538, + 5.5533877379966725, + 5.727661883996916, + 5.888853468997695, + 6.03609833599694, + 6.185815322998678, + 6.366081868996844, + 6.547635423998145, + 6.708500056000048, + 6.884910297998431, + 7.05768466399968, + 7.222783072997117, + 7.403918257998157, + 7.589174346998334, + 7.762550135998026, + 7.925305486998695, + 8.091560280998237, + 8.241832649997377, + 8.36272656499932, + 8.501810291996662, + 8.63839980299963, + 8.795060648997605, + 8.985693653998169, + 9.182162075998349, + 9.390259606996551, + 9.577573134996783, + 9.760819106999406 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 163.61906999918574, + "p95": 200.4434129994479, + "max": 386.39493500159006 + } + }, + { + "start": 28479.119795168, + "first": 28479.4056694, + "last": 28491.068215352, + "end": 28491.068286724, + "elapsed_s": 11.948491555998771, + "chunk_offsets_s": [ + 0.28587423199860496, + 0.5773863760005042, + 0.6690385639994929, + 0.7837948230007896, + 0.9061473439978727, + 1.0111607409999124, + 1.1440003139978216, + 1.2901349399980973, + 1.4448607459999039, + 1.5958548729977338, + 1.7228139680009917, + 1.8622379150001507, + 2.0142565259993717, + 2.1487800579998293, + 2.2892340299986245, + 2.4475727919998462, + 2.6189913659982267, + 2.7874819789976755, + 2.9654183749989897, + 3.1271008269977756, + 3.291798573998676, + 3.4739202040000237, + 3.673183072998654, + 3.8632391889987048, + 4.056747244998405, + 4.244067922998511, + 4.4128347199985, + 4.580321166999056, + 4.730904617001215, + 4.872938290998718, + 5.0370155529999465, + 5.193187996999768, + 5.368097679998755, + 5.552314887998364, + 5.726732285998878, + 5.888201598998421, + 6.035605774999567, + 6.1847852659993805, + 6.365476309998485, + 6.54654018400106, + 6.708088571998815, + 6.883962291998614, + 7.056985921997693, + 7.222115881999343, + 7.404119384998921, + 7.588498728000559, + 7.761574431999179, + 7.925198375000036, + 8.091280009997718, + 8.24080972199954, + 8.362285671999416, + 8.501849696000136, + 8.63829414599968, + 8.793986274999043, + 8.985346681998635, + 9.182184321998648, + 9.388915610998083, + 9.576554795999982, + 9.760039156997664, + 9.892869101000542, + 10.02471628299827, + 10.14232409000033, + 10.285449798000627, + 10.433101474998693, + 10.563438376000704, + 10.688537673999235, + 10.829044520000025, + 10.927935473999241, + 11.012752226000885, + 11.087779417997808, + 11.16760558399983, + 11.257671814000787, + 11.322198785997898, + 11.398135978000937, + 11.466288411000278, + 11.531965195998055, + 11.604973723999137, + 11.681346215998929, + 11.747136821999447, + 11.79315414300072, + 11.840464069999143, + 11.892120294000051, + 11.948420183998678 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 150.0565810019907, + "p95": 193.50805599970045, + "max": 291.5121440018993 + } + }, + { + "start": 28479.120095564, + "first": 28479.500143535, + "last": 28490.13278241, + "end": 28490.132987745, + "elapsed_s": 11.012892180999188, + "chunk_offsets_s": [ + 0.3800479709971114, + 0.5773381139988487, + 0.6689280659993528, + 0.7834055469975283, + 0.9059033339981397, + 1.0112574269987817, + 1.1444122849970881, + 1.2905205409988412, + 1.4448792969997157, + 1.5960542229986459, + 1.723194391997822, + 1.8618108829978155, + 2.0138537689999794, + 2.148682049999479, + 2.288774550997914, + 2.4480649799988896, + 2.6187767809969955, + 2.7876158479994047, + 2.9650575629966625, + 3.127306898997631, + 3.2915975979994982, + 3.4735570249977172, + 3.6743795029979083, + 3.8633524629985914, + 4.05629526499979, + 4.244302508999681, + 4.413662608996674, + 4.580473140998947, + 4.730330251997657, + 4.872964667996712, + 5.0363900319971435, + 5.193584419997933, + 5.368131556999288, + 5.552597322999645, + 5.726753737999388, + 5.88828066899805, + 6.034987671999261, + 6.185444673999882, + 6.36477733799984, + 6.547069595999346, + 6.708130750997952, + 6.88402860199858, + 7.056473507000192, + 7.222050787997432, + 7.403667627997493, + 7.588515383999038, + 7.761473276998004, + 7.924712889998773, + 8.091185923000012, + 8.241063071996905, + 8.362082889998419, + 8.501338096997642, + 8.637743957999191, + 8.793846787997609, + 8.985356478999165, + 9.181608498998685, + 9.389223174999643, + 9.576348159996996, + 9.759832104999077, + 9.892953450998903, + 10.0243357679974, + 10.142352180999296, + 10.284693922996667, + 10.433380061997013, + 10.563553766998666, + 10.688059208998311, + 10.828646398000274, + 10.927389091997611, + 11.01268684599927 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 160.17579199979082, + "p95": 196.2520199995197, + "max": 207.61467600095784 + } + }, + { + "start": 28479.119499234, + "first": 28479.216983772, + "last": 28490.377371318, + "end": 28490.377829596, + "elapsed_s": 11.258330362001288, + "chunk_offsets_s": [ + 0.09748453800057177, + 0.5778236730002391, + 0.6691683960016235, + 0.7838931840014993, + 0.9066385330006597, + 1.0113609830004862, + 1.144720035001228, + 1.2902853520026838, + 1.4445259970016195, + 1.5962386320024962, + 1.7230101480017765, + 1.862118968001596, + 2.0142758729998604, + 2.1489526180012035, + 2.2892278530016483, + 2.44759688799968, + 2.618988452002668, + 2.78787055200155, + 2.9659163300020737, + 3.12814585200249, + 3.2924732669998775, + 3.4744648699997924, + 3.6757800330015016, + 3.864155995001056, + 4.058256858999812, + 4.244573441999819, + 4.413234662999457, + 4.579740667002625, + 4.730720850002399, + 4.873406214999704, + 5.037562204001006, + 5.194328656001744, + 5.369122369000252, + 5.552987761002441, + 5.727577645000565, + 5.888859088001482, + 6.0356662630001665, + 6.1856261160028225, + 6.36596854200252, + 6.547451302001718, + 6.708186497002316, + 6.884045856000739, + 7.057178229002602, + 7.222933162000118, + 7.403571612001542, + 7.589179167000111, + 7.7621613680021255, + 7.9252015960009885, + 8.091262904999894, + 8.241324471000553, + 8.36240681700292, + 8.502096439002344, + 8.638094862999424, + 8.794764856000256, + 8.985588548002852, + 9.18184424699939, + 9.389641133002442, + 9.577187213002617, + 9.760521652002353, + 9.893430850002915, + 10.025192125001922, + 10.143189460002759, + 10.285575179001171, + 10.433688124001492, + 10.56345242200041, + 10.689060963002703, + 10.829441937999945, + 10.9282974380003, + 11.013168514000427, + 11.088409454001521, + 11.167712046000815, + 11.257872084002884 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 156.76645200073835, + "p95": 196.25569899653783, + "max": 480.3391349996673 + } + }, + { + "start": 28479.119171173, + "first": 28479.601879222, + "last": 28489.949025913, + "end": 28489.949196706, + "elapsed_s": 10.83002553299957, + "chunk_offsets_s": [ + 0.48270804899948416, + 0.5779334850012674, + 0.6693761220012675, + 0.7845271120022517, + 0.9071289060011622, + 1.0118616550025763, + 1.1447319149992836, + 1.291046891001315, + 1.4450058000002173, + 1.5968857840016426, + 1.7235010330005025, + 1.8626330730003247, + 2.0150993800016295, + 2.149864909999451, + 2.289981047000765, + 2.448541641999327, + 2.6197679520009842, + 2.788037323000026, + 2.966118058000575, + 3.127613537999423, + 3.292186214999674, + 3.473907929001143, + 3.6739209280021896, + 3.8640231950012094, + 4.058305131002271, + 4.245095256999775, + 4.414432617002603, + 4.5808631729996705, + 4.731122616001812, + 4.873949523000192, + 5.038124750000861, + 5.193740397000511, + 5.368904939001368, + 5.553086454001459, + 5.727400338000734, + 5.889249133000703, + 6.036307648999355, + 6.185618048002652, + 6.366219747000287, + 6.547400695999386, + 6.709298919002322, + 6.884536489000311, + 7.05724704900058, + 7.2226677230028145, + 7.404268010999658, + 7.5897063730008085, + 7.762240981002833, + 7.925742939001793, + 8.09178760100258, + 8.241739383000095, + 8.362955644002795, + 8.50151204299982, + 8.638975999001559, + 8.794574256000487, + 8.985704750000878, + 9.182466509002552, + 9.389591036000638, + 9.577586038001755, + 9.76060754900027, + 9.893261998000526, + 10.025013694001245, + 10.142828801999713, + 10.28625990600267, + 10.43365112399988, + 10.563952027001505, + 10.68886195000232, + 10.829854740000883 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 160.0280374987051, + "p95": 194.2819360010617, + "max": 207.12452699808637 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 107.21459130855922, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 11.988024768998002, + "end_to_end_tps": 106.773219497359, + "timing_valid": true, + "steady_tps": 118.73625897508587, + "per_stream_tps": [ + 26.6, + 22.0, + 24.1, + 23.2, + 23.7 + ], + "ttft_s": [ + 0.19, + 0.38, + 0.48, + 0.29, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19073711900273338, + "decode_tps": 26.600624314031528, + "prefill_tps": 73.39945194306605, + "itl_mean_ms": 37.59310263528331, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.94535599832307, + "round_gap_p50_ms": 164.76493700247374, + "round_gap_p95_ms": 199.76834499902907, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.37980460900143953, + "decode_tps": 21.968864211765652, + "prefill_tps": 36.86105873440556, + "itl_mean_ms": 45.51896676863429, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.56260899922927, + "round_gap_p50_ms": 152.56163800222566, + "round_gap_p95_ms": 193.31580600191955, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.47625333800169756, + "decode_tps": 24.138844756184703, + "prefill_tps": 29.39611942404926, + "itl_mean_ms": 41.42700324313517, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.74355299925082, + "round_gap_p50_ms": 157.1386600007827, + "round_gap_p95_ms": 196.20036799824447, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2864985019987216, + "decode_tps": 23.177464598504333, + "prefill_tps": 48.86587504762056, + "itl_mean_ms": 43.14535767059401, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.63647799903993, + "round_gap_p50_ms": 157.20220399816753, + "round_gap_p95_ms": 198.81788700149627, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09584897099921363, + "decode_tps": 23.697994381111698, + "prefill_tps": 146.0631225776525, + "itl_mean_ms": 42.19766381567894, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.5983319975203, + "round_gap_p50_ms": 158.10372099804226, + "round_gap_p95_ms": 198.66142699902412, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 41.976418826665146, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.85005900159013, + "round_gap_p50_ms": 157.50289200150291, + "round_gap_p95_ms": 198.55956099854666, + "round_gap_mean_ms": 155.13024348984945, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 146.683476844813 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 28491.070230421, + "first": 28491.26096754, + "last": 28500.847208712, + "end": 28500.847324454, + "elapsed_s": 9.777094033001049, + "chunk_offsets_s": [ + 0.19073711900273338, + 0.5735150940017775, + 0.6662146290000237, + 0.7779859280017263, + 0.8952258150020498, + 1.0055705039994791, + 1.140084762002516, + 1.2862956980025047, + 1.4392097380004998, + 1.5923416520017781, + 1.718456917002186, + 1.8559366070003307, + 2.007734185001027, + 2.1429040700022597, + 2.2845314480000525, + 2.4421815280002193, + 2.612232485000277, + 2.781516865001322, + 2.955599307002558, + 3.119637812000292, + 3.2844027490027656, + 3.4663481050010887, + 3.6661164500001178, + 3.851501672001177, + 4.0481133370012685, + 4.234423362002417, + 4.403604690000066, + 4.571667623000394, + 4.725419591002719, + 4.865998982000747, + 5.032182291000936, + 5.189685183002439, + 5.366044651000266, + 5.551688509000087, + 5.72895804899963, + 5.888243974000943, + 6.041014026002813, + 6.189247403999616, + 6.371097463001206, + 6.5501380910027365, + 6.713307846999669, + 6.883345572001417, + 7.060482297001727, + 7.225252931002615, + 7.404576220000308, + 7.592040822000854, + 7.765387863000797, + 7.9296803010001895, + 8.097578059001535, + 8.247738023001148, + 8.370296162000159, + 8.508497328002704, + 8.646810908001498, + 8.80391137100014, + 8.995321116002742, + 9.193880677001289, + 9.40279389600255, + 9.592637112000375, + 9.776978290999978 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 164.76778550168092, + "p95": 199.76834499902907, + "max": 382.7779749990441 + } + }, + { + "start": 28491.070650591, + "first": 28491.4504552, + "last": 28503.057791726, + "end": 28503.057929468, + "elapsed_s": 11.98727887700079, + "chunk_offsets_s": [ + 0.37980460900143953, + 0.5731204150033591, + 0.665687570002774, + 0.7778905530030897, + 0.8950226390006719, + 1.0051785440009553, + 1.1394338610007253, + 1.2860688700020546, + 1.438679464001325, + 1.5918476309998368, + 1.7182709640001121, + 1.8550612920007552, + 2.007622930002981, + 2.142783908002457, + 2.2838526730010926, + 2.4415521690025344, + 2.612285418999818, + 2.7812576860014815, + 2.9546365960013645, + 3.1195269740019285, + 3.2843436440016376, + 3.4657864540022274, + 3.6652994890027912, + 3.851491071000055, + 4.048077948002174, + 4.2338510130030045, + 4.403102401000069, + 4.571404190002795, + 4.724521967000328, + 4.865459800003009, + 5.031048881002789, + 5.188115305001702, + 5.364712135000445, + 5.550657304000197, + 5.728896290002012, + 5.887237596001796, + 6.040329473002203, + 6.188460061002843, + 6.370022670002072, + 6.549860517003253, + 6.71301281200067, + 6.882996213000297, + 7.059728031003033, + 7.22428994800066, + 7.404467387001205, + 7.592132921003213, + 7.765041252001538, + 7.929390496999986, + 8.097225411002, + 8.247387585000979, + 8.369553936001466, + 8.507856993001042, + 8.646563577000052, + 8.803537178002443, + 8.995574649001355, + 9.194591065002896, + 9.402118347003125, + 9.591680320001615, + 9.776108331003343, + 9.90736702200229, + 10.042953701002261, + 10.161802833001275, + 10.306730493000941, + 10.456150835001608, + 10.587670906999847, + 10.714924849002273, + 10.856590734001657, + 10.951187327002117, + 11.040214148000814, + 11.117896100000507, + 11.197323030002735, + 11.287803507002536, + 11.352433114003361, + 11.429098567001347, + 11.497878572001355, + 11.565182584003196, + 11.639210284, + 11.715475255001365, + 11.780557817000954, + 11.826845826002682, + 11.873104887003137, + 11.92508239700328, + 11.987141135003185 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 152.586116000748, + "p95": 193.31580600191955, + "max": 207.5272820002283 + } + }, + { + "start": 28491.070869392, + "first": 28491.54712273, + "last": 28502.111008557, + "end": 28502.11113803, + "elapsed_s": 11.04026863800027, + "chunk_offsets_s": [ + 0.47625333800169756, + 0.5731464010023046, + 0.6660197519995563, + 0.7776070640029502, + 0.8944757170029334, + 1.005234227002802, + 1.1388471190002747, + 1.285970422999526, + 1.4378949990023102, + 1.5923442260027514, + 1.7183301180011767, + 1.8555217940011062, + 2.006961393999518, + 2.142382688001817, + 2.283531283999764, + 2.440883835999557, + 2.6121795370017935, + 2.7809923000022536, + 2.9542804890006664, + 3.1194029250000312, + 3.2838814470014768, + 3.465876793001371, + 3.6662111389996426, + 3.851723898002092, + 4.0479242660003365, + 4.2341094270013855, + 4.4041726930008736, + 4.570933516002697, + 4.7245853429994895, + 4.8655404320015805, + 5.03137471700029, + 5.1885133770010725, + 5.364787351001723, + 5.5509537770012685, + 5.728984273002425, + 5.8874048050020065, + 6.040462724002282, + 6.188466542000242, + 6.370210094999493, + 6.5496861829997215, + 6.7133053979996475, + 6.883238410002377, + 7.059573906000878, + 7.2247097569998004, + 7.404063846999634, + 7.591634737000277, + 7.7650830400016275, + 7.929468144000566, + 8.09758167900145, + 8.247767877001024, + 8.369783362002636, + 8.508149904002494, + 8.646630846000335, + 8.80367101000229, + 8.994869207999727, + 9.193430546001764, + 9.401792138000019, + 9.592215534001298, + 9.77578442000231, + 9.90732269900036, + 10.042885038001259, + 10.161765299002582, + 10.306732207001915, + 10.456128321002325, + 10.587918181001442, + 10.714843264002411, + 10.855438647002302, + 10.951108454002679, + 11.040139165001165 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 157.2456060002878, + "p95": 196.20036799824447, + "max": 208.36159199825488 + } + }, + { + "start": 28491.069996045, + "first": 28491.356494547, + "last": 28502.358560753, + "end": 28502.358776193, + "elapsed_s": 11.288780147999205, + "chunk_offsets_s": [ + 0.2864985019987216, + 0.5742446100011875, + 0.6671371830016142, + 0.7786639180012571, + 0.8957594950006751, + 1.0059236679990136, + 1.1398062789994583, + 1.2866446240004734, + 1.4390970269996615, + 1.5930885349989694, + 1.719408001001284, + 1.85563413500131, + 2.00858013800098, + 2.143499319001421, + 2.283808822001447, + 2.4421249160004663, + 2.6128669460013043, + 2.782129705999978, + 2.9555268149997573, + 3.120534780999151, + 3.2843225319993508, + 3.4669590099983907, + 3.6666277420008555, + 3.8524340149997442, + 4.048056995998195, + 4.234450691001257, + 4.404299200999958, + 4.572166889000073, + 4.725404190001427, + 4.866339192998566, + 5.032137926998985, + 5.189305612999306, + 5.366374404999078, + 5.551536705999752, + 5.729457512999943, + 5.888135444998625, + 6.040769260998786, + 6.189638936000847, + 6.371595982000144, + 6.550722708998364, + 6.713795954001398, + 6.883806090001599, + 7.059989794001012, + 7.225332343001355, + 7.405378027000552, + 7.59258920999855, + 7.766320563998306, + 7.930194180000399, + 8.098200492000615, + 8.248414908000996, + 8.370730364000337, + 8.509120336999331, + 8.64761739000096, + 8.804819593999127, + 8.995823746998212, + 9.194641633999709, + 9.402862251998158, + 9.592578922998655, + 9.777137725999637, + 9.908103409998148, + 10.043837457000336, + 10.162709248001192, + 10.307666611999593, + 10.457080019998102, + 10.588920449001307, + 10.716065726999659, + 10.856633961000625, + 10.951745321999624, + 11.040768923001451, + 11.118662361001043, + 11.19760598799985, + 11.288564708000195 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 157.20220399816753, + "p95": 198.81788700149627, + "max": 287.7461080024659 + } + }, + { + "start": 28491.069904699, + "first": 28491.16575367, + "last": 28501.926157943, + "end": 28501.926815072, + "elapsed_s": 10.856910373000574, + "chunk_offsets_s": [ + 0.09584897099921363, + 0.5737619920000725, + 0.6667085059998499, + 0.7782490089994099, + 0.8956035749979492, + 1.0058326739999757, + 1.1395869829975709, + 1.286999003998062, + 1.4397340949981299, + 1.5929539959979593, + 1.7183255109994207, + 1.855560924999736, + 2.0078166860002966, + 2.143116265997378, + 2.2847524389981118, + 2.4417474119982217, + 2.612844397997833, + 2.7817453409988957, + 2.955698151999968, + 3.1210287069989135, + 3.2848092780004663, + 3.4664076099979866, + 3.6672250230003556, + 3.8520565099970554, + 4.048356139999669, + 4.234876684000483, + 4.4048181229991314, + 4.572192539999378, + 4.725213773999712, + 4.8662855719994695, + 5.032067066997115, + 5.189186456998868, + 5.365956090998225, + 5.551451179999276, + 5.729721440999128, + 5.887825161997171, + 6.04064033800023, + 6.189299115998438, + 6.37060450199715, + 6.5504079409984115, + 6.7136834819975775, + 6.884137177999946, + 7.061174934999144, + 7.224943277997227, + 7.405074155998591, + 7.592800794998766, + 7.765949653999996, + 7.929927354998654, + 8.098016034000466, + 8.248165908000374, + 8.3705429119982, + 8.508554469997762, + 8.647045645000617, + 8.804313528999046, + 8.996462603998225, + 9.195124030997249, + 9.403206334998686, + 9.592166268997971, + 9.776936529997329, + 9.908529124997585, + 10.043787663998955, + 10.162485664997803, + 10.307563767997635, + 10.456641942997521, + 10.588486102999013, + 10.715473396998277, + 10.856253243997344 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 160.68963099860412, + "p95": 198.66142699902412, + "max": 477.91302100085886 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 41.8, + "per_stream_tps": 41.8, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.084, + "decode_tps": 41.824, + "prefill_tps": null, + "itl_mean_ms": 23.909, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.827, + "round_gap_p50_ms": 54.346, + "round_gap_p95_ms": 71.692, + "tokens_per_chunk": 2.349 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 23.909, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.827, + "round_gap_p50_ms": 54.346, + "round_gap_p95_ms": 71.692, + "tokens_per_chunk": 2.349 + }, + "timing_valid": true, + "end_to_end_tps": 41.36243136173511, + "makespan_s": 6.189191292000032, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.396902174005, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.096154842998658, + "end_to_end_tps": 41.99368398491586, + "timing_valid": true, + "per_stream_tps": [ + 42.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08147891299813637, + "decode_tps": 42.396902174005, + "prefill_tps": null, + "itl_mean_ms": 23.586628945100955, + "itl_p50_ms": 0.0, + "itl_p95_ms": 64.80142600048566, + "round_gap_p50_ms": 53.854022997256834, + "round_gap_p95_ms": 72.1925790021487, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.586628945100955, + "itl_p50_ms": 0.0, + "itl_p95_ms": 64.80142600048566, + "round_gap_p50_ms": 53.854022997256834, + "round_gap_p95_ms": 72.1925790021487, + "round_gap_mean_ms": 55.69065167593281, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 28585.773810947, + "first": 28585.85528986, + "last": 28591.869880241, + "end": 28591.86996579, + "elapsed_s": 6.096154842998658, + "chunk_offsets_s": [ + 0.08147891299813637, + 0.1389774609997403, + 0.19674257099904935, + 0.253287506999186, + 0.30591505099801, + 0.35937312500027474, + 0.41459104999739793, + 0.4630321559998265, + 0.5149047479972069, + 0.5758750130007684, + 0.636696238998411, + 0.6806412779988023, + 0.7404093779987306, + 0.8003789089998463, + 0.8505415580002591, + 0.8982122669985984, + 0.946355070998834, + 0.9917275329971744, + 1.0353610069978458, + 1.0841881089982053, + 1.1350675279973075, + 1.1853406059999543, + 1.2392915489981533, + 1.2884095509980398, + 1.344838812998205, + 1.4033098839972808, + 1.4631757930001186, + 1.523221867999382, + 1.5921108570000797, + 1.6431338679976761, + 1.694956524999725, + 1.7416009659973497, + 1.7963195459997223, + 1.8518864269972255, + 1.9117210959993827, + 1.9684464809979545, + 2.0248583469983714, + 2.09705092600052, + 2.1647256430005655, + 2.212993163997453, + 2.2643387829994026, + 2.3123985219972383, + 2.3626197409976157, + 2.4142613090007217, + 2.470598659998359, + 2.5254831109996303, + 2.578962175997731, + 2.6404403149972495, + 2.699554751998221, + 2.7480050009980914, + 2.796426335000433, + 2.8525172349982313, + 2.923254015000566, + 2.984697698000673, + 3.03855172099793, + 3.091512117000093, + 3.1439970819992595, + 3.202419541998097, + 3.258104202999675, + 3.3086579959999654, + 3.3587846650007123, + 3.4151062479977554, + 3.46446181599822, + 3.5112669259979157, + 3.568279970000731, + 3.623792490998312, + 3.668922485998337, + 3.7163275979983155, + 3.7651806639987626, + 3.826187218997802, + 3.881546900000103, + 3.926826307000738, + 3.971802005999052, + 4.020883629000309, + 4.081277818997478, + 4.164522602997749, + 4.246666053997615, + 4.325362417999713, + 4.396346106997953, + 4.4461166500004765, + 4.492769865999435, + 4.539318703999015, + 4.5990378679998685, + 4.671724703999644, + 4.736526130000129, + 4.789510673999757, + 4.835133061998931, + 4.896360213999287, + 4.960705188997963, + 5.0219486040004995, + 5.083297468998353, + 5.130675779997546, + 5.179390455999965, + 5.227820940999663, + 5.273852796999563, + 5.323434398000245, + 5.376718945997709, + 5.441274764998525, + 5.510482517998753, + 5.5616729689973, + 5.612809911999648, + 5.659980424999958, + 5.721104131000175, + 5.801628252000228, + 5.865379701997881, + 5.92932991699854, + 5.994803844998387, + 6.042727394000394, + 6.09606929399888 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 53.90248299772793, + "p95": 72.1925790021487, + "max": 83.24478400027147 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 41.74882831302799, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.192279956001585, + "end_to_end_tps": 41.341800083163825, + "timing_valid": true, + "per_stream_tps": [ + 41.7 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08401615999900969, + "decode_tps": 41.74882831302799, + "prefill_tps": null, + "itl_mean_ms": 23.95276802745488, + "itl_p50_ms": 0.0, + "itl_p95_ms": 67.520966000302, + "round_gap_p50_ms": 54.34587899799226, + "round_gap_p95_ms": 71.69155099836644, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.95276802745488, + "itl_p50_ms": 0.0, + "itl_p95_ms": 67.520966000302, + "round_gap_p50_ms": 54.34587899799226, + "round_gap_p95_ms": 71.69155099836644, + "round_gap_mean_ms": 56.55514673149069, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 28591.870666403, + "first": 28591.954682563, + "last": 28598.06263841, + "end": 28598.062946359, + "elapsed_s": 6.192279956001585, + "chunk_offsets_s": [ + 0.08401615999900969, + 0.14355884900214733, + 0.2027799210009107, + 0.26531201699981466, + 0.31917529700149316, + 0.36807435499940766, + 0.4192550799998571, + 0.4666646869991382, + 0.5193063890001213, + 0.5817122119988198, + 0.6434613160017761, + 0.6890957590003381, + 0.7514057680018595, + 0.8106073189992458, + 0.8625566859991523, + 0.9107009540020954, + 0.959681954998814, + 1.0058351229999971, + 1.0509076939997612, + 1.099320956000156, + 1.1505177020007977, + 1.200463466000656, + 1.2605052889994113, + 1.3120599710018723, + 1.3652241470008448, + 1.4215824909988442, + 1.481315764998726, + 1.543132119000802, + 1.6114584790011577, + 1.6636241439991863, + 1.7175251539993042, + 1.765168370999163, + 1.8170409699996526, + 1.8706131250000908, + 1.9303271800017683, + 1.9893294489993423, + 2.0463261200020497, + 2.116820220999216, + 2.185437913998612, + 2.2363293320013327, + 2.2915996039992024, + 2.341362688999652, + 2.3855657919993973, + 2.43486586400104, + 2.4882072069995047, + 2.5432112170019536, + 2.5967856110000866, + 2.659603160998813, + 2.7202359440016153, + 2.770764846001839, + 2.8176735199995164, + 2.8741896270003053, + 2.9458811779986718, + 3.009068233001017, + 3.0644091940012004, + 3.1187550729991926, + 3.1741785320009512, + 3.2345127039989165, + 3.2975092730011966, + 3.352687965001678, + 3.3997885409989976, + 3.4550649720004003, + 3.5052964230017096, + 3.55368689199895, + 3.6125023679996957, + 3.6696975180020672, + 3.7172933270012436, + 3.7665444949998346, + 3.8153249509996385, + 3.8754658620018745, + 3.933029824998812, + 3.980527787000028, + 4.026330312000937, + 4.0762679489998845, + 4.136076874001446, + 4.2208999850008695, + 4.311619981999684, + 4.393022995998763, + 4.460902143000567, + 4.511187619002158, + 4.558153409001534, + 4.605434899000102, + 4.666408283999772, + 4.740093109001464, + 4.807727183000679, + 4.861889763000363, + 4.907043842002167, + 4.970603329998994, + 5.036931822000042, + 5.099521167001512, + 5.161570660999132, + 5.2086605510012305, + 5.264044569001271, + 5.314916609000647, + 5.364225272001931, + 5.412388843000372, + 5.461572020001768, + 5.525960988001316, + 5.595457228999294, + 5.647843341001135, + 5.701899130999664, + 5.751764511998772, + 5.812576325999544, + 5.893629791000421, + 5.958403689000988, + 6.0226213070018275, + 6.0901422730021295, + 6.1380518459991436, + 6.191972007000004 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 54.67494450022059, + "p95": 71.69155099836644, + "max": 90.71999699881417 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 41.82442755998459, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.189191292000032, + "end_to_end_tps": 41.36243136173511, + "timing_valid": true, + "per_stream_tps": [ + 41.8 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.09221200199681334, + "decode_tps": 41.82442755998459, + "prefill_tps": null, + "itl_mean_ms": 23.90947248628329, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.82669499787153, + "round_gap_p50_ms": 54.47868900228059, + "round_gap_p95_ms": 71.67289599965443, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.90947248628329, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.82669499787153, + "round_gap_p50_ms": 54.47868900228059, + "round_gap_p95_ms": 71.67289599965443, + "round_gap_mean_ms": 56.452921148168876, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 28598.063684911, + "first": 28598.155896913, + "last": 28604.252812397, + "end": 28604.252876203, + "elapsed_s": 6.189191292000032, + "chunk_offsets_s": [ + 0.09221200199681334, + 0.1540355899996939, + 0.21238286500010872, + 0.2711491150002985, + 0.3231990419990325, + 0.37281503699705354, + 0.4240779579995433, + 0.47180707399820676, + 0.5240553409967106, + 0.5868760979974468, + 0.6501845279999543, + 0.6952847439970355, + 0.7560669969971059, + 0.8174136609995912, + 0.868271018996893, + 0.9164789829992515, + 0.9660692969991942, + 1.0108556769991992, + 1.0596068269987882, + 1.1120646349991148, + 1.1643483369989553, + 1.2165944159969513, + 1.2730164099994, + 1.3255798069985758, + 1.3785890569997719, + 1.4375218329987547, + 1.4975906459985708, + 1.55948483499742, + 1.630407953998656, + 1.6826411299989559, + 1.735204355998576, + 1.7825139139968087, + 1.8369955909984128, + 1.890985971000191, + 1.951116757998534, + 2.0091218799971102, + 2.072023877997708, + 2.1484318889997667, + 2.218101103997469, + 2.2662614569999278, + 2.3193692159984494, + 2.36813550199804, + 2.413595466998231, + 2.4644532239981345, + 2.5183637619993533, + 2.5736782120002317, + 2.628496400997392, + 2.69205035199775, + 2.752501659997506, + 2.8041875069975504, + 2.8514658690000942, + 2.908556164999027, + 2.9801736749977863, + 3.0442253279979923, + 3.102609418998327, + 3.1580515949972323, + 3.212530283999513, + 3.272155616999953, + 3.3275238319984055, + 3.378759094997804, + 3.4257890629996837, + 3.479599784997845, + 3.5299707679987478, + 3.576945025997702, + 3.635274879998178, + 3.691558549999172, + 3.736497771998984, + 3.785252464000223, + 3.8343137339979876, + 3.8938591409969376, + 3.9508648039991385, + 3.9965802659971814, + 4.043709024997952, + 4.098406127999624, + 4.162636574998032, + 4.245964852998441, + 4.3282692579996365, + 4.4067806849998306, + 4.473607379997702, + 4.522416926000005, + 4.569030787999509, + 4.6153043569975125, + 4.677334445997985, + 4.74900734199764, + 4.8159764089978125, + 4.8703477879971615, + 4.917930263000017, + 4.980675049999263, + 5.0467826099993545, + 5.112958374997106, + 5.177618066998548, + 5.224880171997938, + 5.2739155540002685, + 5.320979211999656, + 5.367198062998796, + 5.414955549000297, + 5.46233516799839, + 5.526393899999675, + 5.595766501999606, + 5.646486927998922, + 5.699157773997285, + 5.746946159997606, + 5.808189350998873, + 5.887041245998262, + 5.9510207989987975, + 6.014053308997973, + 6.085560281997459, + 6.134589846999006, + 6.189127485999052 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 54.48018300194235, + "p95": 71.67289599965443, + "max": 83.32827800040832 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 60.9, + "per_stream_tps": 30.9, + "ttft_s_max": 0.16, + "request": { + "ttft_s": 0.122, + "decode_tps": 30.925, + "prefill_tps": null, + "itl_mean_ms": 32.339, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.215, + "round_gap_p50_ms": 75.357, + "round_gap_p95_ms": 94.103, + "tokens_per_chunk": 2.349 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 32.199, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.971, + "round_gap_p50_ms": 75.216, + "round_gap_p95_ms": 94.162, + "tokens_per_chunk": 2.349 + }, + "timing_valid": true, + "end_to_end_tps": 60.56364457654716, + "makespan_s": 8.453916596001363, + "steady_tps": 62.2, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 60.60593203362275, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 8.496254711000802, + "end_to_end_tps": 60.26184682729337, + "timing_valid": true, + "steady_tps": 61.71420612130676, + "per_stream_tps": [ + 30.6, + 31.2 + ], + "ttft_s": [ + 0.16, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16038739800205803, + "decode_tps": 30.591566299190355, + "prefill_tps": null, + "itl_mean_ms": 32.68874794509839, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.9063599970541, + "round_gap_p50_ms": 74.95926299816347, + "round_gap_p95_ms": 94.28701899742009, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08111930200175266, + "decode_tps": 31.153076449296922, + "prefill_tps": null, + "itl_mean_ms": 32.09955850195233, + "itl_p50_ms": 0.0, + "itl_p95_ms": 89.83929299938609, + "round_gap_p50_ms": 76.0956209996948, + "round_gap_p95_ms": 95.4809569993813, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 32.394153223525365, + "itl_p50_ms": 0.0, + "itl_p95_ms": 89.1499509998539, + "round_gap_p50_ms": 75.21635400189552, + "round_gap_p95_ms": 95.4809569993813, + "round_gap_mean_ms": 76.48619511110155, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 28604.253735418, + "first": 28604.414122816, + "last": 28612.749753542, + "end": 28612.749871099, + "elapsed_s": 8.49613568100176, + "chunk_offsets_s": [ + 0.16038739800205803, + 0.2291294040005596, + 0.3123907270019117, + 0.40494250700066914, + 0.4938366780006618, + 0.5671422330015048, + 0.6320646580024913, + 0.693700269999681, + 0.7642856870006653, + 0.838966209001228, + 0.9240464280010201, + 0.9913801360016805, + 1.066108091003116, + 1.1344923819997348, + 1.2028197460022056, + 1.2683926870013238, + 1.3299881020029716, + 1.3879651030001696, + 1.4465344030031702, + 1.5081294960000378, + 1.5711190290021477, + 1.6384016560004966, + 1.711815500002558, + 1.7816067870007828, + 1.8488798010002938, + 1.9278389589999279, + 2.0087529370030097, + 2.079587023003114, + 2.164770398001565, + 2.2633258380010375, + 2.340716763999808, + 2.4093612370015762, + 2.475320023000677, + 2.553120212000067, + 2.646357290002925, + 2.7378460760010057, + 2.8171226080012275, + 2.897798836002039, + 2.984170094001456, + 3.0546701470011612, + 3.1255949110018264, + 3.2045002690028923, + 3.2921832660031214, + 3.3864702850005415, + 3.471796752000955, + 3.5502054070020677, + 3.6259139659996436, + 3.7105077900014294, + 3.798089282001456, + 3.8879842970018217, + 3.9767867310001748, + 4.072837900002924, + 4.157365694001783, + 4.237029051000718, + 4.320815095001308, + 4.405519994001224, + 4.483618638001644, + 4.5650040520013135, + 4.640488668999751, + 4.702422308000678, + 4.765582638003252, + 4.846292103000451, + 4.918058926003141, + 4.985012204000668, + 5.071166224002809, + 5.157124420002219, + 5.215688162003062, + 5.282157154000743, + 5.365189318003104, + 5.454095678000158, + 5.530733071002032, + 5.59744884400061, + 5.665219986000011, + 5.734356484001182, + 5.809572838003078, + 5.900303980000899, + 6.00073133800106, + 6.1093497649999335, + 6.223240039002121, + 6.304504668001755, + 6.372576208999817, + 6.432795634002105, + 6.506053771001461, + 6.582075168000301, + 6.656390685002407, + 6.737557468000887, + 6.804604570002994, + 6.866452715003106, + 6.94141197800127, + 7.022823231000075, + 7.094650890001503, + 7.162235777002934, + 7.223665923000226, + 7.285805649000395, + 7.347723689999839, + 7.410048571000516, + 7.475993351999932, + 7.556590888001665, + 7.6359181240004546, + 7.710031072001584, + 7.789728171002935, + 7.875104941002064, + 7.9672492040008365, + 8.032941665002, + 8.103297224002745, + 8.185322036002617, + 8.266978349001874, + 8.33064992800064, + 8.396379997000622, + 8.442947622002976, + 8.496018124002148 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 75.0878085000295, + "p95": 94.28701899742009, + "max": 113.89027400218765 + } + }, + { + "start": 28604.253616388, + "first": 28604.33473569, + "last": 28612.520123108, + "end": 28612.520233923, + "elapsed_s": 8.266617535002297, + "chunk_offsets_s": [ + 0.08111930200175266, + 0.22896655699878465, + 0.31275933500000974, + 0.40561440199962817, + 0.4943499030014209, + 0.5674079800010077, + 0.6324271580015193, + 0.6940083900008176, + 0.7641998330000206, + 0.8394146630016621, + 0.9239843470022606, + 0.9918000630023016, + 1.0664853870002844, + 1.134421774000657, + 1.2031838090006204, + 1.2683311700020568, + 1.3298976699988998, + 1.387731499002257, + 1.4464336080018256, + 1.5084978310005681, + 1.5713818170006562, + 1.6388429059988994, + 1.7117680010014737, + 1.7816408700018656, + 1.8488979729991115, + 1.927873765998811, + 2.008757292001974, + 2.080087326001376, + 2.1651217569997243, + 2.2633231389991124, + 2.3410927779987105, + 2.409373612001218, + 2.4758300360008434, + 2.5534035230011796, + 2.6466472310021345, + 2.7383355390011275, + 2.81715911900028, + 2.8981048370005738, + 2.984438569001213, + 3.05495123700166, + 3.125410127999203, + 3.2049199100001715, + 3.292584867998812, + 3.3869466169999214, + 3.4718008220006595, + 3.5496590080001624, + 3.6257978870016814, + 3.710442995001358, + 3.798148844001844, + 3.88798813700123, + 3.977138088001084, + 4.072619045000465, + 4.1577214730023115, + 4.236810903999867, + 4.321202265000466, + 4.405914931001462, + 4.483189119000599, + 4.565366832001018, + 4.6401602670011926, + 4.702264110001124, + 4.765806989002158, + 4.846095846001845, + 4.918460730001243, + 4.986239923000539, + 5.0717366400022, + 5.157525580001675, + 5.216000589000032, + 5.282090984001115, + 5.365526093999506, + 5.454537260000507, + 5.530991462001111, + 5.5977684350000345, + 5.665524380001443, + 5.734782837000239, + 5.8099892790014565, + 5.900185004000377, + 6.000033602998883, + 6.109110083001724, + 6.223096260000602, + 6.3044786390019, + 6.372513988000719, + 6.4332181319987285, + 6.506377779001923, + 6.582473400001618, + 6.656152925999777, + 6.737569927001459, + 6.805016852002154, + 6.86629914399964, + 6.941245047000848, + 7.022676241998852, + 7.095064422999712, + 7.162612780000927, + 7.224071026001184, + 7.285779811001703, + 7.348099007998826, + 7.410273288998724, + 7.476378930001374, + 7.55700109800091, + 7.635616635001497, + 7.710508344000118, + 7.790146568000637, + 7.87495622500137, + 7.967180855001061, + 8.033307061999949, + 8.103592478000792, + 8.185784878001868, + 8.266506719999597 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 76.11725000060687, + "p95": 95.4809569993813, + "max": 147.847254997032 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 60.915050007482805, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 8.453916596001363, + "end_to_end_tps": 60.56364457654716, + "timing_valid": true, + "steady_tps": 62.16283301428583, + "per_stream_tps": [ + 30.5, + 31.7 + ], + "ttft_s": [ + 0.08, + 0.16 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08145445000263862, + "decode_tps": 30.457525003741402, + "prefill_tps": null, + "itl_mean_ms": 32.83260868626587, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.25133500047377, + "round_gap_p50_ms": 75.75545399959083, + "round_gap_p95_ms": 94.16187199894921, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16348519399980432, + "decode_tps": 31.67934879722374, + "prefill_tps": null, + "itl_mean_ms": 31.566305431367844, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.71552999658161, + "round_gap_p50_ms": 75.92698900043615, + "round_gap_p95_ms": 94.04456699849106, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 32.19945705881686, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.8137729994487, + "round_gap_p50_ms": 75.92698900043615, + "round_gap_p95_ms": 94.16187199894921, + "round_gap_mean_ms": 76.02649583331758, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 28612.750729104, + "first": 28612.832183554, + "last": 28621.204498769, + "end": 28621.204567387, + "elapsed_s": 8.453838283003279, + "chunk_offsets_s": [ + 0.08145445000263862, + 0.23104841400345322, + 0.3143301110030734, + 0.4104917430013302, + 0.5008544140000595, + 0.5765624410014425, + 0.6388079330026812, + 0.7001197140016302, + 0.7688675590034109, + 0.845369908001885, + 0.9244527110022318, + 0.9877850670018233, + 1.0635405210014142, + 1.134149054003501, + 1.2024664160016982, + 1.267519278000691, + 1.324395176001417, + 1.3860744910016365, + 1.4459865800017724, + 1.507887131003372, + 1.5720890230004443, + 1.63755176199993, + 1.7115975560009247, + 1.7814169400007813, + 1.8469457360006345, + 1.9191030020010658, + 1.996966706999956, + 2.066208786000061, + 2.151843199000723, + 2.2403776420032955, + 2.3183154860016657, + 2.390014329001133, + 2.4587212200021895, + 2.538004888003343, + 2.629799210000783, + 2.721042684002896, + 2.802787967000768, + 2.8794449690030888, + 2.9630632050029817, + 3.0326438440024504, + 3.104157732002932, + 3.183681966002041, + 3.2704957390014897, + 3.3644175560002623, + 3.4561023410024063, + 3.5347942280022835, + 3.608072027000162, + 3.6922651270033384, + 3.7787327750011173, + 3.865984110001591, + 3.950745394002297, + 4.03500280300068, + 4.119162252001843, + 4.197225582000101, + 4.2786324430016975, + 4.362660231003247, + 4.444150801002252, + 4.529655084003025, + 4.60457364100148, + 4.66723920000004, + 4.731071281999903, + 4.813313148002635, + 4.881752119999874, + 4.945183851003094, + 5.029628982003487, + 5.115683876003459, + 5.173672158001864, + 5.239778503000707, + 5.321441799002059, + 5.4156036710010085, + 5.495183547001943, + 5.561950200000865, + 5.629919042999973, + 5.699306014001195, + 5.774777167000138, + 5.86088622400348, + 5.956372863001889, + 6.064834840002732, + 6.167085623001185, + 6.249229447003017, + 6.316480808000051, + 6.37954195200291, + 6.455621979002899, + 6.53314716100067, + 6.603868805003003, + 6.6859210150032595, + 6.753230723003071, + 6.817220717002783, + 6.887357966003037, + 6.965371506001247, + 7.037169516002905, + 7.106402147001063, + 7.168517482001334, + 7.231857452003169, + 7.2932682810023834, + 7.3565652780016535, + 7.428684331000113, + 7.512338815002295, + 7.590438767001615, + 7.666786244000832, + 7.746308905003389, + 7.829701801001647, + 7.908943439000723, + 7.975511356002244, + 8.048170079000556, + 8.131443737001973, + 8.212895014999958, + 8.275889775002724, + 8.341233794002619, + 8.392407085000741, + 8.453769665000436 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 75.91774049978994, + "p95": 94.16187199894921, + "max": 149.5939640008146 + } + }, + { + "start": 28612.750650791, + "first": 28612.914135985, + "last": 28620.96354387, + "end": 28620.96377063, + "elapsed_s": 8.213119839001592, + "chunk_offsets_s": [ + 0.16348519399980432, + 0.23088782400009222, + 0.3140682890007156, + 0.41083349299879046, + 0.5011665279998851, + 0.5762460160003684, + 0.6381525280012283, + 0.7006013979989802, + 0.7686973259988008, + 0.845717224998225, + 0.9241939860003185, + 0.9879778499998793, + 1.0631434300012188, + 1.1346860410012596, + 1.2029008850004175, + 1.267775950000214, + 1.3247103329995298, + 1.3864073090007878, + 1.446282455999608, + 1.508206763999624, + 1.572469786999136, + 1.6377205940007116, + 1.7112254239982576, + 1.781181413000013, + 1.8462309060014377, + 1.9188007599987031, + 1.9961908420009422, + 2.0665569820012024, + 2.1523191449996375, + 2.240162802998384, + 2.318741556999157, + 2.3901594029994158, + 2.4585486209980445, + 2.53782892199888, + 2.6302275259986345, + 2.7214023460001044, + 2.8025027260009665, + 2.87948062400028, + 2.9635242039985314, + 3.0328458809999574, + 3.103923863000091, + 3.1840815840005234, + 3.2703875219995098, + 3.364432088998001, + 3.456052131998149, + 3.5351139529993816, + 3.6084170680005627, + 3.6927890820006724, + 3.778720645001158, + 3.866309976001503, + 3.951181150998309, + 4.0350183459995606, + 4.1191572889983945, + 4.197538686999906, + 4.278942326000106, + 4.3628810990012425, + 4.444333548999566, + 4.52991905700037, + 4.6048109240000485, + 4.667443124999409, + 4.731242200999986, + 4.813647556999058, + 4.881323321998934, + 4.945408405998023, + 5.029605503998027, + 5.115669678998529, + 5.1732501149999734, + 5.240361290998408, + 5.321129575000668, + 5.41574836899963, + 5.495384420999471, + 5.5622832990011375, + 5.63060613700145, + 5.69916786199974, + 5.77444942800139, + 5.861164957997971, + 5.95666839399928, + 6.0650199109986715, + 6.167410958998516, + 6.24917589100005, + 6.316161079001176, + 6.379699411998445, + 6.455626400998881, + 6.532797064999613, + 6.604166578999866, + 6.686281862999749, + 6.7535704170004465, + 6.816973692999454, + 6.887610710000445, + 6.965359359001013, + 7.037342796000303, + 7.106428866998613, + 7.168530597999052, + 7.232248001000698, + 7.293714733001252, + 7.356384856000659, + 7.42889649800054, + 7.512632746998861, + 7.590836766001303, + 7.667055387999426, + 7.746538119998149, + 7.83024182599911, + 7.909229374999995, + 7.975794376001431, + 8.04843412199989, + 8.131406538999727, + 8.212893078998604 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 76.07280549927964, + "p95": 94.04456699849106, + "max": 108.3515169993916 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 61.391914523459434, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 8.391077104002761, + "end_to_end_tps": 61.017196440223714, + "timing_valid": true, + "steady_tps": 62.568193719661835, + "per_stream_tps": [ + 30.7, + 31.9 + ], + "ttft_s": [ + 0.08, + 0.16 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08350224499736214, + "decode_tps": 30.695957261729717, + "prefill_tps": null, + "itl_mean_ms": 32.577579890194635, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.97148500141338, + "round_gap_p50_ms": 74.54883199898177, + "round_gap_p95_ms": 92.747866998252, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16413096000178484, + "decode_tps": 31.885928236280897, + "prefill_tps": null, + "itl_mean_ms": 31.361796733336615, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.17962200171314, + "round_gap_p50_ms": 74.85322000138694, + "round_gap_p95_ms": 91.96125499875052, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 31.969688311765626, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.97148500141338, + "round_gap_p50_ms": 74.57076400169171, + "round_gap_p95_ms": 92.18740500000422, + "round_gap_mean_ms": 75.48398629166884, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 28621.205648414, + "first": 28621.289150659, + "last": 28629.596433531, + "end": 28629.596493632, + "elapsed_s": 8.390845218000322, + "chunk_offsets_s": [ + 0.08350224499736214, + 0.232132097997237, + 0.31542999600060284, + 0.40946735800025635, + 0.49555718899864587, + 0.5684302890003892, + 0.6308935220004059, + 0.6930595149970031, + 0.762489351000113, + 0.8369194219994824, + 0.9154779399977997, + 0.9848289289984677, + 1.061329157997534, + 1.1302125109978078, + 1.1970408759989368, + 1.2604845860005298, + 1.3174995389999822, + 1.3745415499979572, + 1.4312955489986052, + 1.4926403289973678, + 1.5550016140005027, + 1.6208745689982607, + 1.6950162059983995, + 1.7635683829976188, + 1.8301452809973853, + 1.9039241449972906, + 1.9866926350005087, + 2.0597827290002897, + 2.145844048998697, + 2.2322179900002084, + 2.3096186439979647, + 2.379500177998125, + 2.444403222998517, + 2.5196526090003317, + 2.6124004759985837, + 2.704129595997074, + 2.78431455199825, + 2.8611503769971023, + 2.9481218619985157, + 3.0206619709970255, + 3.091184990997135, + 3.170961970998178, + 3.2575640639988706, + 3.3489717779993953, + 3.434642806998454, + 3.511955496000155, + 3.586227300998871, + 3.671257860998594, + 3.7587810849981906, + 3.844655905999389, + 3.9312280929989356, + 4.023162517998571, + 4.10993736599994, + 4.1889432839998335, + 4.271682206999685, + 4.35734478899758, + 4.435191402997589, + 4.517526958999952, + 4.591592297998432, + 4.6547360209988256, + 4.718185095000081, + 4.7988684060001106, + 4.865393264997692, + 4.933208725997247, + 5.022117294000054, + 5.108638464997057, + 5.165518611000152, + 5.232119139000133, + 5.312083033997624, + 5.402103599997645, + 5.477367334999144, + 5.543148524000571, + 5.611840339999617, + 5.680410838998796, + 5.754895079997368, + 5.839497175998986, + 5.935831507998955, + 6.0471758489984495, + 6.1481906219996745, + 6.229036344997439, + 6.295257546000357, + 6.35633601399968, + 6.427654759998404, + 6.502225524000096, + 6.571841850000055, + 6.65268166199894, + 6.7184795279972604, + 6.779694150998694, + 6.850198660999013, + 6.928103179998288, + 7.00265201199727, + 7.0697851209988585, + 7.131031028999132, + 7.193151732997649, + 7.254468423998333, + 7.316341268000542, + 7.383391993997066, + 7.464406003000477, + 7.542904969999654, + 7.618582466999214, + 7.696190640999703, + 7.7791250469999795, + 7.856441157997324, + 7.922270668997953, + 7.997213113998441, + 8.080938398998114, + 8.161639909998485, + 8.22525191099703, + 8.290184790999774, + 8.33669514599751, + 8.390785116996994 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 74.55979800033674, + "p95": 92.747866998252, + "max": 148.62985299987486 + } + }, + { + "start": 28621.205416528, + "first": 28621.369547488, + "last": 28629.366805655, + "end": 28629.366908222, + "elapsed_s": 8.161491694001597, + "chunk_offsets_s": [ + 0.16413096000178484, + 0.2327616719994694, + 0.3155147620018397, + 0.41003683600138174, + 0.49621814700003597, + 0.5685432680002123, + 0.631187562001287, + 0.6929344910022337, + 0.7625291980002658, + 0.8373824180016527, + 0.9158133260025352, + 0.9851468699998804, + 1.061075241999788, + 1.1306636530025571, + 1.1975541730025725, + 1.2603899340028875, + 1.3180983800011745, + 1.3752844520022336, + 1.4317529550025938, + 1.4926346650026971, + 1.5555680470024527, + 1.6213394350015733, + 1.695128374001797, + 1.7642136909998953, + 1.830710755999462, + 1.9039866260027338, + 1.987012155001139, + 2.0602704690027167, + 2.146608882001601, + 2.2328256239998154, + 2.310230330000195, + 2.3798177910030063, + 2.444818017000216, + 2.520527870001388, + 2.6124891250001383, + 2.7046765300001425, + 2.7848011830028554, + 2.861678875000507, + 2.9488584970022202, + 3.0207080640029744, + 3.091650448001019, + 3.1716467270016437, + 3.2581225450012425, + 3.349417262001225, + 3.435127532000479, + 3.512474913000915, + 3.586846988000616, + 3.6716057320008986, + 3.7592420740002126, + 3.8452691540005617, + 3.9318826620001346, + 4.023115973002859, + 4.110710659002507, + 4.189523635999649, + 4.272431844001403, + 4.357907493002131, + 4.435909891002666, + 4.518109714001184, + 4.592057068999566, + 4.654817914000887, + 4.718963704002817, + 4.798785380000481, + 4.8662095880026754, + 4.933206943002006, + 5.022418011001719, + 5.109238463002839, + 5.166051748001337, + 5.232566390001011, + 5.312491609001881, + 5.402772200002801, + 5.4778274900018005, + 5.5439055650022055, + 5.61254031900171, + 5.6807657800018205, + 5.755311738001183, + 5.840242708000005, + 5.935787168000388, + 6.047095262001676, + 6.148213466000016, + 6.229500151999673, + 6.295769451000524, + 6.356812841000647, + 6.42826372700074, + 6.502244371000415, + 6.572332913001446, + 6.6530496230006975, + 6.719378941001196, + 6.780139288999635, + 6.850616738000099, + 6.92857211700175, + 7.002956391002954, + 7.070088833999762, + 7.131421065001632, + 7.193605827000283, + 7.254383424002299, + 7.316342686001008, + 7.3833524260007835, + 7.464990696000314, + 7.543463253001391, + 7.618672995002271, + 7.696529880002345, + 7.779465681000147, + 7.8564195330000075, + 7.922650472999521, + 7.997520139000699, + 8.081266895002045, + 8.161389127002622 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 74.86144300128217, + "p95": 91.96125499875052, + "max": 111.30809400128783 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.1, + "per_stream_tps": 21.1, + "ttft_s_max": 0.34, + "request": { + "ttft_s": 0.212, + "decode_tps": 21.125, + "prefill_tps": null, + "itl_mean_ms": 47.339, + "itl_p50_ms": 0.0, + "itl_p95_ms": 122.855, + "round_gap_p50_ms": 109.477, + "round_gap_p95_ms": 130.335, + "tokens_per_chunk": 2.339 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 46.565, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.827, + "round_gap_p50_ms": 109.348, + "round_gap_p95_ms": 130.427, + "tokens_per_chunk": 2.359 + }, + "timing_valid": true, + "end_to_end_tps": 81.80902778815613, + "makespan_s": 12.516955985000095, + "steady_tps": 85.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.22059936065065, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 12.493965665002179, + "end_to_end_tps": 81.9595657180655, + "timing_valid": true, + "steady_tps": 84.99556931748361, + "per_stream_tps": [ + 21.0, + 21.5, + 21.0, + 22.5 + ], + "ttft_s": [ + 0.25, + 0.16, + 0.33, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.24549678999755997, + "decode_tps": 20.998129022483056, + "prefill_tps": null, + "itl_mean_ms": 47.62329057647388, + "itl_p50_ms": 0.0, + "itl_p95_ms": 123.2583489982062, + "round_gap_p50_ms": 108.77855199942132, + "round_gap_p95_ms": 130.3227570024319, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16485110599751351, + "decode_tps": 21.54769347359211, + "prefill_tps": null, + "itl_mean_ms": 46.40867948235645, + "itl_p50_ms": 0.0, + "itl_p95_ms": 123.35363499732921, + "round_gap_p50_ms": 109.49736000111443, + "round_gap_p95_ms": 130.42696900083683, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3277964629996859, + "decode_tps": 20.96103405444139, + "prefill_tps": null, + "itl_mean_ms": 47.70757002744872, + "itl_p50_ms": 0.0, + "itl_p95_ms": 122.24133700146922, + "round_gap_p50_ms": 108.69205899871304, + "round_gap_p95_ms": 130.0618620007299, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08825008500207332, + "decode_tps": 22.542073691901077, + "prefill_tps": null, + "itl_mean_ms": 44.36149103528484, + "itl_p50_ms": 0.0, + "itl_p95_ms": 123.63686599928769, + "round_gap_p50_ms": 109.98684100195533, + "round_gap_p95_ms": 130.51682299919776, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 46.52525778039097, + "itl_p50_ms": 0.0, + "itl_p95_ms": 122.54573300015181, + "round_gap_p50_ms": 109.20210000040242, + "round_gap_p95_ms": 130.42696900083683, + "round_gap_mean_ms": 110.3622393860437, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 28629.599455102, + "first": 28629.844951892, + "last": 28641.988890989, + "end": 28641.989047953, + "elapsed_s": 12.389592850999179, + "chunk_offsets_s": [ + 0.24549678999755997, + 0.4391565459991398, + 0.5714609979986562, + 0.7034778719971655, + 0.8148213979984575, + 0.9129131449990382, + 1.0052203060004103, + 1.1130322999997588, + 1.2333440259972122, + 1.3370146159977594, + 1.4352329209978052, + 1.530947996998293, + 1.6391803329970571, + 1.7459332260004885, + 1.8527971980001894, + 1.9736278599993966, + 2.0819786749998457, + 2.1992741049980395, + 2.299668092000502, + 2.3979085279970604, + 2.507648071998119, + 2.6173055569997814, + 2.7232985279988497, + 2.833929858999909, + 2.933370692997414, + 3.041824235999229, + 3.1534875699981058, + 3.2559499829985725, + 3.374388807998912, + 3.496158130998083, + 3.613651656000002, + 3.7160605149983894, + 3.813150914000289, + 3.917205832000036, + 4.03372364399911, + 4.162700580000092, + 4.273674983000092, + 4.38645242399798, + 4.502146705999621, + 4.6078513549982745, + 4.718196877998707, + 4.8272044309997, + 4.93929174999721, + 5.0546252929998445, + 5.169227853999473, + 5.275264687999879, + 5.379466037000384, + 5.49399537499994, + 5.619900232999498, + 5.732191488998069, + 5.8422999899994466, + 5.953181709999626, + 6.076440058997832, + 6.202453810998122, + 6.3430018839972035, + 6.46678759399947, + 6.581740244997491, + 6.697840861997975, + 6.818148000998917, + 6.920916289996967, + 7.036848798998108, + 7.16717155600054, + 7.27896368599977, + 7.395424768998055, + 7.5217620539988275, + 7.640579582999635, + 7.741313590999198, + 7.8422507739996945, + 7.952607256996998, + 8.073978255997645, + 8.179715184000088, + 8.285891867999453, + 8.394639824997284, + 8.51323896899703, + 8.617012667000381, + 8.732495647000178, + 8.849397212998156, + 8.9853945750001, + 9.115416290998837, + 9.221151825997367, + 9.315685415996995, + 9.406944081998518, + 9.51034434399844, + 9.618512294997345, + 9.715381453999726, + 9.828637071997946, + 9.939149637997616, + 10.034720425999694, + 10.133303766997415, + 10.231860141997458, + 10.335207456999342, + 10.429647998000291, + 10.520743064000271, + 10.629521615999693, + 10.728353761998733, + 10.833867633999034, + 10.95163211199906, + 11.065695690998837, + 11.175218951997522, + 11.28308936599933, + 11.400556274998962, + 11.496302917999856, + 11.595903542998713, + 11.68914233599935, + 11.793078085000161, + 11.899692323997442, + 12.001290797998081, + 12.075708639000368, + 12.157391518998338, + 12.246906275999208, + 12.32132482399902, + 12.389435886998399 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 108.77855199942132, + "p95": 130.3227570024319, + "max": 193.6597560015798 + } + }, + { + "start": 28629.600919461, + "first": 28629.765770567, + "last": 28641.599983835, + "end": 28641.60016303, + "elapsed_s": 11.9992435690001, + "chunk_offsets_s": [ + 0.16485110599751351, + 0.4376594119967194, + 0.5699623900000006, + 0.7025030379991222, + 0.8135727549997682, + 0.9112335459976748, + 1.0032412149994343, + 1.1113819860001968, + 1.2319715630001156, + 1.3355930209982034, + 1.433823934999964, + 1.5295215909973194, + 1.6377731729990046, + 1.744309671998053, + 1.851550519997545, + 1.9724239939969266, + 2.080592460999469, + 2.197924262000015, + 2.298048608998215, + 2.3964017639991653, + 2.5062497519975295, + 2.6160241999969003, + 2.72122648399818, + 2.832275282999035, + 2.9317869869992137, + 3.040155386999686, + 3.1523226350000186, + 3.253994602997409, + 3.3727723539996077, + 3.4953180869997595, + 3.6121234499987622, + 3.7145175480000034, + 3.8115463419999287, + 3.915886488997785, + 4.032440916998894, + 4.160787970999081, + 4.272341253999912, + 4.385171171998081, + 4.500782749997597, + 4.606443946999207, + 4.71680339699742, + 4.8255693319988495, + 4.938107144997048, + 5.053830842000025, + 5.167984563999198, + 5.273910697997053, + 5.378137050996884, + 5.4933073239990335, + 5.618361450997327, + 5.7305508899989945, + 5.840285905997007, + 5.95142674499948, + 6.07478037999681, + 6.201237121000304, + 6.3415851529971405, + 6.465659116998722, + 6.58000453799832, + 6.696583953998925, + 6.816385284997523, + 6.9201295120001305, + 7.036242095997295, + 7.1658880849972775, + 7.278180784996948, + 7.394012729997485, + 7.520351035000203, + 7.639609717996791, + 7.7393108819997, + 7.840491191996989, + 7.9513541249980335, + 8.072697018997133, + 8.177574946999812, + 8.284622153998498, + 8.393336922999879, + 8.511654729998554, + 8.61531757299963, + 8.730936030999146, + 8.848123192998173, + 8.98412605699923, + 9.114553026000067, + 9.219594792997668, + 9.31432874299935, + 9.405385580997972, + 9.50864991999697, + 9.616835244996764, + 9.713532664998638, + 9.826663861997076, + 9.937605224997242, + 10.033339438999974, + 10.132110057998943, + 10.230564764999144, + 10.333959347997734, + 10.42813143999956, + 10.519134633999784, + 10.628336734000186, + 10.727299034999305, + 10.832454837000114, + 10.95050908399935, + 11.064095542998984, + 11.173592903000099, + 11.281421995998244, + 11.399610153999674, + 11.494803855999635, + 11.59471557500001, + 11.687724076997256, + 11.791536276999977, + 11.898146044997702, + 11.999064373998408 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 109.61618799956341, + "p95": 130.42696900083683, + "max": 272.8083059992059 + } + }, + { + "start": 28629.600048784, + "first": 28629.927845247, + "last": 28642.093275604, + "end": 28642.093340922, + "elapsed_s": 12.493292138002289, + "chunk_offsets_s": [ + 0.3277964629996859, + 0.43906165299995337, + 0.5705963149994204, + 0.7025562220005668, + 0.8138794430014968, + 0.9120556480011146, + 1.0037644600015483, + 1.1123923750019458, + 1.2328103059990099, + 1.3361511680013791, + 1.4345550130019546, + 1.5302695329992275, + 1.6388840950021404, + 1.7452829960020608, + 1.852079367999977, + 1.9734997109990218, + 2.0812174929997127, + 2.198423896999884, + 2.299133773001813, + 2.39689953999914, + 2.5069763040010002, + 2.616790549000143, + 2.722151723999559, + 2.8330932240023685, + 2.9328194160007115, + 3.041171746001055, + 3.1526763010006107, + 3.2550943919995916, + 3.3736917520000134, + 3.4959330890014826, + 3.6128309300002, + 3.7156969940006093, + 3.8122644479990413, + 3.9169483250007033, + 4.033513808000862, + 4.161871774002066, + 4.273492576001445, + 4.385925863000011, + 4.501619460999791, + 4.6070677540010365, + 4.718009840002196, + 4.8262534480018076, + 4.938482298999588, + 5.054345840999304, + 5.168496666999999, + 5.274612699999125, + 5.37933544200132, + 5.493996601002436, + 5.6194659089997, + 5.731371526999283, + 5.841980462999345, + 5.95284472700223, + 6.075907072001428, + 6.202184362999105, + 6.342484623000928, + 6.466260343000613, + 6.580938682000124, + 6.697312894000788, + 6.817010459999437, + 6.9200620480005455, + 7.036360221001814, + 7.166422083002544, + 7.278561464001541, + 7.39492072700159, + 7.521424235001177, + 7.640322365001339, + 7.740519857001345, + 7.841304143999878, + 7.952414252999006, + 8.073439095001959, + 8.178884944001766, + 8.285245111001132, + 8.393937169999845, + 8.512233240002388, + 8.616287078999449, + 8.73175889599952, + 8.848919824002223, + 8.984344304000842, + 9.115118666999479, + 9.220413319002546, + 9.315386345999286, + 9.405937891002395, + 9.509249819002434, + 9.618148598001426, + 9.714484896001522, + 9.827706628999294, + 9.93864287100223, + 10.034465946002456, + 10.133287425000162, + 10.231369662000361, + 10.33477900700018, + 10.428897925001365, + 10.520104778999666, + 10.629005172002508, + 10.727545216999715, + 10.833358084000793, + 10.951428099000623, + 11.064797813000041, + 11.174511151999468, + 11.282009721999202, + 11.400092320000113, + 11.495587969002372, + 11.59525906899944, + 11.688839555001323, + 11.792680630001996, + 11.899415593001322, + 12.000631015002, + 12.075433441001223, + 12.15667203600242, + 12.246210778001114, + 12.320964775000903, + 12.38895861399942, + 12.442294084001333, + 12.493226819999109 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 108.69205899871304, + "p95": 130.0618620007299, + "max": 140.3002600018226 + } + }, + { + "start": 28629.599375257, + "first": 28629.687625342, + "last": 28640.999805556, + "end": 28641.000160035, + "elapsed_s": 11.400784778001253, + "chunk_offsets_s": [ + 0.08825008500207332, + 0.4400136210024357, + 0.5714340480008104, + 0.7031429590024345, + 0.8148080589999154, + 0.9132727159994829, + 1.004949514001055, + 1.1123728769998706, + 1.2333042159989418, + 1.3370107110022218, + 1.4347648020011547, + 1.5311466230014048, + 1.639709925999341, + 1.745549380000739, + 1.8524555139993026, + 1.9739029630000005, + 2.0817075760023727, + 2.198819965000439, + 2.29947456300215, + 2.3977742030001536, + 2.507554579002317, + 2.617681964999065, + 2.722538034002355, + 2.83334524699967, + 2.932751479002036, + 3.0416542929997377, + 3.153111453000747, + 3.2551605600019684, + 3.373772139999346, + 3.4960974780005927, + 3.6135794069996336, + 3.7159203789997264, + 3.813286397999036, + 3.9173778850017698, + 4.033920207999472, + 4.1621780450004735, + 4.273847686999943, + 4.386446286000137, + 4.501870522999525, + 4.608108561998961, + 4.718135334002, + 4.826530328002264, + 4.938792683999054, + 5.054427531998954, + 5.1689772339996125, + 5.275398363999557, + 5.3799512360019435, + 5.4942537839997385, + 5.6196599290015, + 5.731916528999136, + 5.841903370001091, + 5.952734349000821, + 6.076760995001678, + 6.202413175000402, + 6.343399716999556, + 6.467036582998844, + 6.581329276999895, + 6.698071909002465, + 6.817297210000106, + 6.920400744002109, + 7.036371336002048, + 7.166888159001246, + 7.278799384999729, + 7.395158792001894, + 7.522154944999784, + 7.640928921999148, + 7.740639538998948, + 7.84228375100065, + 7.95226232099958, + 8.07418458200118, + 8.179017729999032, + 8.28577418200075, + 8.394421985001827, + 8.513033138999162, + 8.61724624999988, + 8.732145248999586, + 8.8497884899989, + 8.984884362998855, + 9.115191371001856, + 9.220968518999143, + 9.31559971200113, + 9.406349478002085, + 9.510345681999752, + 9.618897026000923, + 9.714748276001046, + 9.828973576000863, + 9.939529582999967, + 10.034476798999094, + 10.13313535999987, + 10.231817171999865, + 10.3344604720005, + 10.429177633999643, + 10.520879545001662, + 10.629496918001678, + 10.72813829900042, + 10.833796386999893, + 10.951417208001658, + 11.066079062999052, + 11.174919117001991, + 11.282090386001073, + 11.400430298999709 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 110.00680650249706, + "p95": 130.51682299919776, + "max": 351.76353600036236 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.07315489616512, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 12.516955985000095, + "end_to_end_tps": 81.80902778815613, + "timing_valid": true, + "steady_tps": 85.68727693352723, + "per_stream_tps": [ + 20.7, + 21.7, + 20.7, + 23.0 + ], + "ttft_s": [ + 0.09, + 0.26, + 0.18, + 0.34 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08871566699963296, + "decode_tps": 20.69025344725101, + "prefill_tps": null, + "itl_mean_ms": 48.331935737252365, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.14709400056745, + "round_gap_p50_ms": 108.58812899823533, + "round_gap_p95_ms": 128.88782400113996, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25985388399931253, + "decode_tps": 21.69047474830053, + "prefill_tps": null, + "itl_mean_ms": 46.10318638038805, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.30945300022722, + "round_gap_p50_ms": 109.34785400240798, + "round_gap_p95_ms": 128.7743680004496, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17871510999975726, + "decode_tps": 20.66781841070506, + "prefill_tps": null, + "itl_mean_ms": 48.3844003333241, + "itl_p50_ms": 0.0, + "itl_p95_ms": 124.94388399863965, + "round_gap_p50_ms": 108.30005599927972, + "round_gap_p95_ms": 128.823225000815, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3447773620027874, + "decode_tps": 23.019465104609946, + "prefill_tps": null, + "itl_mean_ms": 43.44149594508767, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.21302199739148, + "round_gap_p50_ms": 110.21724099919084, + "round_gap_p95_ms": 128.03684900063672, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 46.565254599013045, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.21302199739148, + "round_gap_p50_ms": 109.34785400240798, + "round_gap_p95_ms": 128.823225000815, + "round_gap_mean_ms": 110.45711556044955, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 28642.096624318, + "first": 28642.185339985, + "last": 28654.509983598, + "end": 28654.510153426, + "elapsed_s": 12.413529107998329, + "chunk_offsets_s": [ + 0.08871566699963296, + 0.4577386229975673, + 0.5866264469987073, + 0.7135767180006951, + 0.8261849459995574, + 0.9254619019993697, + 1.0197281209984794, + 1.1361025259975577, + 1.2565662630004226, + 1.3617792849981925, + 1.4607075489984709, + 1.5586829489984666, + 1.6610807569995814, + 1.767392536999978, + 1.8754360309976619, + 1.9965716570004588, + 2.110918334998132, + 2.2310618459996476, + 2.332122330997663, + 2.43151732700062, + 2.5418804679975437, + 2.6454583189988625, + 2.74882832199728, + 2.8597635019978043, + 2.9592321860000084, + 3.072900061997643, + 3.1901245289991493, + 3.291870357999869, + 3.409697741997661, + 3.5363191979995463, + 3.6489044039990404, + 3.749498377997952, + 3.848629419997451, + 3.9512191339999845, + 4.070305148998159, + 4.203019906999543, + 4.3171293060004245, + 4.4283362360001775, + 4.545989062000444, + 4.6471641049975005, + 4.7542439850003575, + 4.862832113998593, + 4.977241254997352, + 5.0972243920004985, + 5.21783650100042, + 5.322453797998605, + 5.426062147998891, + 5.542425761999766, + 5.662362977000157, + 5.771855595998204, + 5.882794189998094, + 5.992322990998218, + 6.1202715639992675, + 6.247445375000098, + 6.38779766999869, + 6.514073733000259, + 6.6246852399999625, + 6.736580549997598, + 6.856778461999056, + 6.96025827899939, + 7.07945018099781, + 7.2045972749983775, + 7.316342476999125, + 7.4323426419978205, + 7.552702705997945, + 7.6636736679975, + 7.765334333998908, + 7.866706401000556, + 7.9756705010004225, + 8.10199986799853, + 8.209223244997702, + 8.314915390998067, + 8.422422900999663, + 8.532318178000423, + 8.630789259997982, + 8.743568722999044, + 8.863221769999654, + 8.999583265998808, + 9.136215583999729, + 9.243765016999532, + 9.338283876997593, + 9.42970694199903, + 9.537868330997298, + 9.640742054998555, + 9.734308572998998, + 9.848094945999037, + 9.961425244997372, + 10.05533206399923, + 10.161042438998265, + 10.25918504999936, + 10.360068755999237, + 10.454804887998762, + 10.546666255999298, + 10.649108470999636, + 10.745312596998701, + 10.851926556999388, + 10.96913788099846, + 11.085495245999482, + 11.196576460999495, + 11.303158720998908, + 11.42187358099909, + 11.518515183997806, + 11.615054459998646, + 11.705219938998198, + 11.809342355998524, + 11.915864966998925, + 12.016039540998463, + 12.097421472997667, + 12.18211524599974, + 12.269775513999775, + 12.34373632599818, + 12.413359279998986 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 108.58812899823533, + "p95": 128.88782400113996, + "max": 369.02295599793433 + } + }, + { + "start": 28642.096958187, + "first": 28642.356812071, + "last": 28654.113124598, + "end": 28654.113615521, + "elapsed_s": 12.01665733399932, + "chunk_offsets_s": [ + 0.25985388399931253, + 0.45768649299861863, + 0.5864608609990682, + 0.71352865299923, + 0.8260592549995636, + 0.9255168719973881, + 1.0194486439977481, + 1.135832757998287, + 1.2565328429991496, + 1.3620097589991929, + 1.4607519629971648, + 1.558652275998611, + 1.6611496839977917, + 1.7672883809973428, + 1.8753905409976142, + 1.996310716996959, + 2.1106606379980803, + 2.230680778997339, + 2.3317189349982073, + 2.4312297939977725, + 2.541890959997545, + 2.6453433019996737, + 2.748848260998784, + 2.8596866659972875, + 2.9590880010000546, + 3.0723738579981728, + 3.1896285000002536, + 3.2912836899995455, + 3.4093145429978904, + 3.5357442069980607, + 3.6487971999995352, + 3.7492826899979264, + 3.848575545998756, + 3.951175484999112, + 4.06979005500034, + 4.2028555369979586, + 4.317187987999205, + 4.428241779998643, + 4.546160317997419, + 4.646935204000329, + 4.754148326999712, + 4.862672316998214, + 4.97666855100033, + 5.097159029999602, + 5.216589142997691, + 5.321955219998927, + 5.426384270998824, + 5.541904749999958, + 5.662265049999405, + 5.771168799998122, + 5.882276992997504, + 5.992434815998422, + 6.1193526679999195, + 6.247345650997886, + 6.3875348339970515, + 6.513691043997824, + 6.624589031998767, + 6.736509613998351, + 6.856368502998521, + 6.959844643999531, + 7.078795635999995, + 7.2041050890002225, + 7.31641673699778, + 7.4319524689999525, + 7.552283061999333, + 7.66307942200001, + 7.764948087999073, + 7.866062155997497, + 7.975410009999905, + 8.10140354000032, + 8.209141526000167, + 8.314271145998646, + 8.42185866699947, + 8.532493722999789, + 8.630329251998774, + 8.743198009997286, + 8.863067195998156, + 8.999620103997586, + 9.135803503999341, + 9.242909805998352, + 9.337892591996933, + 9.42949116499949, + 9.537312856999051, + 9.640770196998346, + 9.734333906999382, + 9.84811471699868, + 9.96152630099823, + 10.05538471699765, + 10.160936790998676, + 10.25879791799889, + 10.359676284999296, + 10.454507831997034, + 10.546587670000008, + 10.648429161999957, + 10.7453113019983, + 10.85102323299725, + 10.968867988998682, + 11.085624220999307, + 11.196063835999666, + 11.302682191999338, + 11.421300911999424, + 11.518384752998827, + 11.614949574999628, + 11.70514252999783, + 11.809195851998084, + 11.915820570997312, + 12.016166410998267 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 109.75283850166306, + "p95": 128.7743680004496, + "max": 197.8326089993061 + } + }, + { + "start": 28642.096539809, + "first": 28642.275254919, + "last": 28654.613277004, + "end": 28654.613379959, + "elapsed_s": 12.516840149997734, + "chunk_offsets_s": [ + 0.17871510999975726, + 0.458285671997146, + 0.587108896997961, + 0.7143854229980207, + 0.8261694069988152, + 0.9258374849996471, + 1.0197386919971905, + 1.1364551769984246, + 1.257246845998452, + 1.362185130998114, + 1.4609841530000267, + 1.5596828889974859, + 1.6618625539995264, + 1.7677657280000858, + 1.8760657839993655, + 1.9971670220002125, + 2.1109158279978146, + 2.231251835997682, + 2.332768295000278, + 2.4317632189995493, + 2.542167775998678, + 2.646188409999013, + 2.749047625999083, + 2.860039147999487, + 2.9594476879974536, + 3.0731785360003414, + 3.1901282379985787, + 3.2914084529984393, + 3.4100287729997945, + 3.5369099130002724, + 3.6498944710001524, + 3.7501593479973963, + 3.8494461329974, + 3.9518759019993013, + 4.069944462997228, + 4.203733712998655, + 4.317800835997332, + 4.428887699999905, + 4.546334824997757, + 4.647643018997769, + 4.755119154997374, + 4.863590590997774, + 4.977406423997309, + 5.09770579799806, + 5.216609043000062, + 5.322737034999591, + 5.426407110997388, + 5.542037141000037, + 5.663068355999712, + 5.771301415999915, + 5.882515416000388, + 5.992660837997391, + 6.120076143997721, + 6.247243434998381, + 6.3882078120004735, + 6.514492312999209, + 6.625655366999126, + 6.736766367997916, + 6.856434497000009, + 6.960883149997244, + 7.079443945000094, + 7.204387828998733, + 7.3170291669994185, + 7.433053831999132, + 7.553469480997592, + 7.663317604998156, + 7.765158477999648, + 7.866072212997096, + 7.976079435000429, + 8.101741283000592, + 8.209650401000545, + 8.315326531999744, + 8.422606888998416, + 8.53274488599709, + 8.630402634000347, + 8.743299803998525, + 8.863541118997091, + 8.999606171997584, + 9.136422004998167, + 9.24365184799899, + 9.338659697998082, + 9.42945042700012, + 9.5379116969998, + 9.641526977000467, + 9.735044286997436, + 9.849318384000071, + 9.961729645998275, + 10.055487850997451, + 10.160927261000325, + 10.259337879997474, + 10.360014624999167, + 10.45510105699941, + 10.547383778997755, + 10.649532649997127, + 10.745545185000083, + 10.852302507999411, + 10.969725710998318, + 11.08574827999837, + 11.196726384998328, + 11.303190219998214, + 11.4221592899994, + 11.519107505999273, + 11.61545223199937, + 11.705848270998104, + 11.80995350199737, + 11.916944158998376, + 12.016811785997561, + 12.097236636000162, + 12.181850475997635, + 12.270118496999203, + 12.343621015999815, + 12.413226097000006, + 12.465201783998054, + 12.516737194997404 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 108.30005599927972, + "p95": 128.823225000815, + "max": 279.5705619973887 + } + }, + { + "start": 28642.096423974, + "first": 28642.441201336, + "last": 28653.518782802, + "end": 28653.519155501, + "elapsed_s": 11.422731526999996, + "chunk_offsets_s": [ + 0.3447773620027874, + 0.4584998740028823, + 0.5873245360016881, + 0.7142504270013887, + 0.8268630630009284, + 0.926134061999619, + 1.0200420550027047, + 1.1362521190021653, + 1.2574651409995568, + 1.3626568760009832, + 1.461371532001067, + 1.5599582579998241, + 1.6621901110011095, + 1.7682929600014177, + 1.8763717370020458, + 1.9970171750028385, + 2.111246681000921, + 2.2316303340012382, + 2.3326537950015336, + 2.431992377001734, + 2.542235631000949, + 2.6463706890026515, + 2.749336339002184, + 2.8601059730026464, + 2.9597473760004505, + 3.0727561349995085, + 3.1900606990020606, + 3.2924342549995345, + 3.4104126630008977, + 3.5376856309994764, + 3.6496173280029325, + 3.750038788999518, + 3.849623223002709, + 3.9524751490025665, + 4.069864853001491, + 4.203460607001034, + 4.318051197002205, + 4.428936448002787, + 4.546404854001594, + 4.647668153000268, + 4.755128715001774, + 4.863864566999837, + 4.977620321002178, + 5.0975417330009805, + 5.216785565000464, + 5.32290949900198, + 5.426748406000115, + 5.542242552000971, + 5.663133235000714, + 5.771804952000821, + 5.882586425999762, + 5.992720629001269, + 6.120757478001906, + 6.24726246100181, + 6.388228547999461, + 6.51454430600279, + 6.625488317000418, + 6.7369605300009425, + 6.856751096001972, + 6.960738465000759, + 7.079861725000228, + 7.204416771000979, + 7.317529765001382, + 7.432913941000152, + 7.553270873002475, + 7.663488114001666, + 7.765019796999695, + 7.865961982002773, + 7.97612946800291, + 8.10232754900062, + 8.209996409001178, + 8.315368365001632, + 8.422678962000646, + 8.53278655900067, + 8.63055968900153, + 8.743453170001885, + 8.863718158001575, + 8.999598193000566, + 9.13649952600099, + 9.243663962002756, + 9.338702901000943, + 9.429631686001812, + 9.537981641002261, + 9.64168792000055, + 9.735088028999598, + 9.849372676002531, + 9.961789197001053, + 10.055693283000437, + 10.161157517999527, + 10.259562957002345, + 10.359653221999906, + 10.455159313001786, + 10.547432459999982, + 10.649545828000555, + 10.745764482002414, + 10.852210860000923, + 10.969948909001687, + 11.08579339300195, + 11.196922197999811, + 11.303122790999623, + 11.422358828000142 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 110.23024749920296, + "p95": 128.03684900063672, + "max": 140.96608699765056 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.57186808147307, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 12.587439913997514, + "end_to_end_tps": 81.35093450267748, + "timing_valid": true, + "steady_tps": 84.39135219663441, + "per_stream_tps": [ + 20.7, + 21.3, + 20.8, + 22.7 + ], + "ttft_s": [ + 0.16, + 0.08, + 0.33, + 0.25 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16240649499741266, + "decode_tps": 20.71189603826327, + "prefill_tps": null, + "itl_mean_ms": 48.281431992155355, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.70224700093968, + "round_gap_p50_ms": 109.70977300166851, + "round_gap_p95_ms": 131.3653390025138, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.0825779210026667, + "decode_tps": 21.252217971794636, + "prefill_tps": null, + "itl_mean_ms": 47.05391227057678, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.90604499846813, + "round_gap_p50_ms": 110.0595030002296, + "round_gap_p95_ms": 131.7330000019865, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.32878729299773113, + "decode_tps": 20.80198551953058, + "prefill_tps": null, + "itl_mean_ms": 48.07233420392104, + "itl_p50_ms": 0.0, + "itl_p95_ms": 122.07186500017997, + "round_gap_p50_ms": 109.45622800136334, + "round_gap_p95_ms": 130.34708600025624, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.24591895299818134, + "decode_tps": 22.698465083602827, + "prefill_tps": null, + "itl_mean_ms": 44.05584238039035, + "itl_p50_ms": 0.0, + "itl_p95_ms": 122.45121899832156, + "round_gap_p50_ms": 110.9993089994532, + "round_gap_p95_ms": 132.23732199912774, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 46.86588021176088, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.82742900040466, + "round_gap_p50_ms": 110.0595030002296, + "round_gap_p95_ms": 131.7330000019865, + "round_gap_mean_ms": 111.1702274790607, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 28654.616514123, + "first": 28654.778920618, + "last": 28667.090685776, + "end": 28667.090807398, + "elapsed_s": 12.474293274997763, + "chunk_offsets_s": [ + 0.16240649499741266, + 0.44076046099871746, + 0.5758571989972552, + 0.7086536629976763, + 0.8197167250000348, + 0.918077327998617, + 1.0128825139981927, + 1.1225922869998612, + 1.244561829000304, + 1.3472096499972395, + 1.4464830389988492, + 1.5447583610002766, + 1.6526522179992753, + 1.758599469998444, + 1.864543395997316, + 1.98599486799867, + 2.095052284999838, + 2.2118261979994713, + 2.315241177999269, + 2.413212436997128, + 2.5247429789997113, + 2.636609935998422, + 2.7416017679970537, + 2.8508542019990273, + 2.9525269939986174, + 3.0632302909980353, + 3.175961384000402, + 3.277151584999956, + 3.3972604400005366, + 3.5181757459977234, + 3.6377439989992126, + 3.738948062000418, + 3.837116127997433, + 3.9403410499980964, + 4.055976116997044, + 4.185125481999421, + 4.298393318000308, + 4.411154673998681, + 4.52757465799732, + 4.63494793299833, + 4.745055957999284, + 4.853644196999085, + 4.969262153997988, + 5.08616311500009, + 5.201552577000257, + 5.3091451289983524, + 5.4139493179973215, + 5.530203858997993, + 5.660502539998561, + 5.772669878999295, + 5.885770368000522, + 5.996897650999017, + 6.12182434100032, + 6.2487091519979, + 6.390594800999679, + 6.516577799997322, + 6.635331471999962, + 6.751044413998898, + 6.871697917998972, + 6.979556894999405, + 7.098263415999099, + 7.219378216999758, + 7.333062649999192, + 7.4511923889986065, + 7.572865226997237, + 7.691131121999206, + 7.792541686998447, + 7.896875262998947, + 8.007710772999417, + 8.129413020000356, + 8.236779849998129, + 8.342465595000249, + 8.45122412300043, + 8.564673758999561, + 8.669420807997085, + 8.782171913997445, + 8.903692410000076, + 9.042667064997659, + 9.174032404000172, + 9.280765693998546, + 9.376244176997716, + 9.470030675998714, + 9.581341099998099, + 9.688971380997828, + 9.784753923999233, + 9.898898739997094, + 10.0125171519976, + 10.107731496998895, + 10.206505154998013, + 10.306845617997169, + 10.408629802997893, + 10.50478791599744, + 10.602558317998046, + 10.708313998999074, + 10.804874599998584, + 10.911626441000408, + 11.031130464998569, + 11.14488792100019, + 11.254697650998423, + 11.36126379399866, + 11.480497546999686, + 11.580066295999131, + 11.67929302599805, + 11.771751432999736, + 11.87580381099906, + 11.981672603000334, + 12.081249561997538, + 12.157284051998431, + 12.240341392000119, + 12.329077968999627, + 12.4040793819986, + 12.474171652997029 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 109.70977300166851, + "p95": 131.3653390025138, + "max": 278.3539660013048 + } + }, + { + "start": 28654.616363495, + "first": 28654.698941416, + "last": 28666.697689045, + "end": 28666.698037712, + "elapsed_s": 12.081674217002728, + "chunk_offsets_s": [ + 0.0825779210026667, + 0.4405900609999662, + 0.5757040490025247, + 0.7086746490022051, + 0.8196689329997753, + 0.9184799659997225, + 1.0124790020017826, + 1.121974734000105, + 1.2438021630005096, + 1.3474250930012204, + 1.4458689610000874, + 1.5443729590006114, + 1.6527220429998124, + 1.7580134980016737, + 1.8648720510027488, + 1.9855740370003332, + 2.094616793001478, + 2.211625191001076, + 2.315460461002658, + 2.4137096960002964, + 2.524403990999417, + 2.636263128999417, + 2.741376026002399, + 2.8514355290026288, + 2.952298870000959, + 3.0635593510014587, + 3.176301704999787, + 3.278219195002748, + 3.3968809390025854, + 3.518465606000973, + 3.638116442001774, + 3.739438519001851, + 3.8366953740005556, + 3.939925957001833, + 4.055688872002065, + 4.184572387002845, + 4.297799834999751, + 4.411101233999943, + 4.527352266002708, + 4.6353241720025835, + 4.744413890002761, + 4.8530954720008594, + 4.969785738001519, + 5.085742303999723, + 5.200791721999849, + 5.309093433999806, + 5.413492118001159, + 5.530196161002095, + 5.661072172002605, + 5.77235081300023, + 5.886040850000427, + 5.996774993000145, + 6.1217562630008615, + 6.24863805700079, + 6.390395943002659, + 6.516957578001893, + 6.635194974001934, + 6.75106616300036, + 6.871686059999774, + 6.979912046001118, + 7.098179440999957, + 7.21917244100041, + 7.332693409000058, + 7.45095980500264, + 7.572865850001108, + 7.691477145002864, + 7.7926279050007, + 7.89672344200153, + 8.00758130000031, + 8.129804546999367, + 8.236848550001014, + 8.342572919002123, + 8.451520464001078, + 8.564679416002036, + 8.6693387430023, + 8.782598426001641, + 8.904400616000203, + 9.042681631999585, + 9.174414632001572, + 9.281185957999696, + 9.37670863600215, + 9.470468047002214, + 9.58186868300254, + 9.689001500999439, + 9.785435826001049, + 9.899258749999717, + 10.012572957002703, + 10.107732013999339, + 10.207428396002797, + 10.307229491001635, + 10.408985758000199, + 10.504857144002017, + 10.60317343800125, + 10.708778063002683, + 10.80496667099942, + 10.911839444001089, + 11.03148252200117, + 11.144988127001852, + 11.255012657002226, + 11.361940875001892, + 11.480955940001877, + 11.579699565001647, + 11.679521256002772, + 11.771158497002034, + 11.87575661200026, + 11.981417967999732, + 12.081325549999747 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 110.37689899967518, + "p95": 131.7330000019865, + "max": 358.0121399972995 + } + }, + { + "start": 28654.616019811, + "first": 28654.944807104, + "last": 28667.203252326, + "end": 28667.203320528, + "elapsed_s": 12.58730071699756, + "chunk_offsets_s": [ + 0.32878729299773113, + 0.44151704699834227, + 0.5762267930003873, + 0.7088866119993327, + 0.8201489739985846, + 0.9183735879996675, + 1.0131718349985022, + 1.1226280629998655, + 1.2446999280000455, + 1.3478414429991972, + 1.4467457649989228, + 1.544809021997935, + 1.6532129759980307, + 1.7588751139992382, + 1.8651140349975321, + 1.9866709800007811, + 2.0952599960000953, + 2.2121601609978825, + 2.315517054998054, + 2.4141995629979647, + 2.5249793010007124, + 2.6369640929988236, + 2.741780177999317, + 2.8515428389982844, + 2.9527892650003196, + 3.063344700000016, + 3.176201120000769, + 3.278325794999546, + 3.397822864997579, + 3.518569648000266, + 3.638378453997575, + 3.7393356319989834, + 3.8373821809982473, + 3.940544236000278, + 4.056075582000631, + 4.185221403997275, + 4.298499856999115, + 4.411396917999809, + 4.528008294000756, + 4.635763856000267, + 4.745460271999036, + 4.853767426000559, + 4.970239842998126, + 5.086371747998783, + 5.201455103997432, + 5.309207926999079, + 5.414117530999647, + 5.530815772999631, + 5.661162858999887, + 5.772759690997191, + 5.8861285879975185, + 5.997029271999054, + 6.12224984599743, + 6.248842425997282, + 6.390871985000558, + 6.517226086998562, + 6.6357654460007325, + 6.750964265000221, + 6.872067314998276, + 6.980357683998591, + 7.098673507000058, + 7.218968429999222, + 7.333369225998467, + 7.451353112999641, + 7.573635810000269, + 7.691031652997481, + 7.793303996997565, + 7.896653034000337, + 8.007721961999778, + 8.130269633998978, + 8.237380008999025, + 8.343370932998369, + 8.451916087000427, + 8.565113855998788, + 8.66975734999869, + 8.78305733399975, + 8.904656691000127, + 9.042981607999536, + 9.175212228998134, + 9.280834534998576, + 9.377247544998681, + 9.470049469000514, + 9.582276135999564, + 9.689582863997202, + 9.784963572998095, + 9.899199714000133, + 10.01308192400029, + 10.107801546997507, + 10.206541837000259, + 10.307821578997391, + 10.409486008000385, + 10.504852660000324, + 10.603293413998472, + 10.70859982500042, + 10.804915261000133, + 10.911803675997362, + 11.031874517997494, + 11.144895976998669, + 11.254929974998959, + 11.362016341998242, + 11.481190859998605, + 11.580924711997795, + 11.680062137998902, + 11.771837504998985, + 11.875900200000615, + 11.98225781200017, + 12.081803156997921, + 12.158038967998436, + 12.24110050300078, + 12.329364648998308, + 12.404878900000767, + 12.47540797200054, + 12.527974117998383, + 12.587232514997595 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 109.45622800136334, + "p95": 130.34708600025624, + "max": 142.02955900327652 + } + }, + { + "start": 28654.615880614, + "first": 28654.861799567, + "last": 28666.096039374, + "end": 28666.096419333, + "elapsed_s": 11.480538718999014, + "chunk_offsets_s": [ + 0.24591895299818134, + 0.44117351900058566, + 0.5763053970003966, + 0.7090917249988706, + 0.8200713969999924, + 0.9187776969993138, + 1.0131434840004658, + 1.122817509000015, + 1.2446213999974134, + 1.3477590619986586, + 1.4466891409974778, + 1.545569145000627, + 1.6534048859975883, + 1.7588449979994039, + 1.8650948549984605, + 1.9864798489979876, + 2.0954567179978767, + 2.2119477550004376, + 2.315450783997221, + 2.4137043669979903, + 2.5247036759974435, + 2.637019189998682, + 2.7416421189991524, + 2.8520599240000593, + 2.953345570000238, + 3.0637521520002338, + 3.176193921997765, + 3.2783658129992546, + 3.3975504109985195, + 3.5190931549987, + 3.6381136419986433, + 3.739415973999712, + 3.8374650009973266, + 3.940632491998258, + 4.056024429999525, + 4.185320851000142, + 4.2985993519978365, + 4.41166971200073, + 4.5280566129986255, + 4.636102750999271, + 4.745148071000585, + 4.854138603997853, + 4.970419342000241, + 5.086463952997292, + 5.201644229997328, + 5.309642984000675, + 5.414212633000716, + 5.530897764998372, + 5.661597753998649, + 5.772987108997768, + 5.885947875998681, + 5.997333827999682, + 6.1223062989993196, + 6.249055551998026, + 6.391155690998858, + 6.517658865999692, + 6.635843140000361, + 6.751318793998507, + 6.872094800997729, + 6.98063651999837, + 7.098448251999798, + 7.219383284998912, + 7.33340897999733, + 7.451260816000286, + 7.573712034998607, + 7.691444325999328, + 7.7936270350001, + 7.896851711997442, + 8.007921901000373, + 8.130475988000399, + 8.23758166799962, + 8.343424521997804, + 8.451662980998663, + 8.564878003999183, + 8.669947824000701, + 8.783314500000415, + 8.904489532000298, + 9.043063684999652, + 9.17530100699878, + 9.281112878998101, + 9.377425566999591, + 9.470440839999355, + 9.582095648998802, + 9.689804323999851, + 9.785031873998378, + 9.899297907999426, + 10.013279552997119, + 10.108012332999351, + 10.206967261998216, + 10.307835252999212, + 10.40967797799749, + 10.504922554999212, + 10.60359950199927, + 10.708834565997677, + 10.805115189999924, + 10.911987768999097, + 11.03186375400037, + 11.145100286998058, + 11.255168841998966, + 11.361551291000069, + 11.480158759997721 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 111.03474900119181, + "p95": 132.23732199912774, + "max": 195.25456600240432 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 87.0, + "per_stream_tps": 18.0, + "ttft_s_max": 0.45, + "request": { + "ttft_s": 0.264, + "decode_tps": 18.047, + "prefill_tps": null, + "itl_mean_ms": 55.41, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.343, + "round_gap_p50_ms": 125.854, + "round_gap_p95_ms": 146.982, + "tokens_per_chunk": 2.286 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 54.343, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.336, + "round_gap_p50_ms": 125.639, + "round_gap_p95_ms": 146.997, + "tokens_per_chunk": 2.315 + }, + "timing_valid": true, + "end_to_end_tps": 86.75180171391649, + "makespan_s": 14.75473678599883, + "steady_tps": 91.2, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 86.02171343316907, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 14.919701069000439, + "end_to_end_tps": 85.79260362391128, + "timing_valid": true, + "steady_tps": 90.40344101384008, + "per_stream_tps": [ + 17.7, + 18.7, + 17.7, + 19.4, + 17.6 + ], + "ttft_s": [ + 0.1, + 0.35, + 0.27, + 0.18, + 0.45 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.09771646999797667, + "decode_tps": 17.707985116872944, + "prefill_tps": null, + "itl_mean_ms": 56.471698694119425, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.00270100243506, + "round_gap_p50_ms": 127.13789099871065, + "round_gap_p95_ms": 147.29692500259262, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3508735110008274, + "decode_tps": 18.672982059395878, + "prefill_tps": null, + "itl_mean_ms": 53.55331016862514, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.08473699889146, + "round_gap_p50_ms": 127.58775699694525, + "round_gap_p95_ms": 147.2003279995988, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.26807844300128636, + "decode_tps": 17.74228662280006, + "prefill_tps": null, + "itl_mean_ms": 56.362520866669534, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.26003500132356, + "round_gap_p50_ms": 127.05676599944127, + "round_gap_p95_ms": 147.00724999784143, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.18264704600005643, + "decode_tps": 19.44412176755162, + "prefill_tps": null, + "itl_mean_ms": 51.429424890189765, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.7444550022774, + "round_gap_p50_ms": 128.10411999817006, + "round_gap_p95_ms": 146.9823789993825, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.44714204800038715, + "decode_tps": 17.61962567480908, + "prefill_tps": null, + "itl_mean_ms": 56.75489470980692, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.04257999861147, + "round_gap_p50_ms": 125.93085899789003, + "round_gap_p95_ms": 146.34454699989874, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 54.91436986588216, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.90166999812936, + "round_gap_p50_ms": 127.35797200002708, + "round_gap_p95_ms": 146.9967680022819, + "round_gap_mean_ms": 127.76609777189735, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 28667.204906798, + "first": 28667.302623268, + "last": 28681.702906435, + "end": 28681.703047122, + "elapsed_s": 14.498140323998086, + "chunk_offsets_s": [ + 0.09771646999797667, + 0.5856410699998378, + 0.7320019760009018, + 0.8746077539981343, + 1.0062912850007706, + 1.1238782590007759, + 1.2326268600008916, + 1.353005797998776, + 1.483105011000589, + 1.6013610810005048, + 1.716974767998181, + 1.827674427000602, + 1.941080802000215, + 2.0682186929989257, + 2.202770285999577, + 2.3435677099987515, + 2.4840023510005267, + 2.6212084319995483, + 2.738228418998915, + 2.853963339999609, + 2.985473518998333, + 3.1073386339994613, + 3.225885002000723, + 3.360433859001205, + 3.482124787999055, + 3.607207971999742, + 3.735368549998384, + 3.85218893199999, + 3.9873332280003524, + 4.125693295998644, + 4.253379591998964, + 4.378161280001223, + 4.492589445999329, + 4.608378073000495, + 4.7337649740002234, + 4.878487555000902, + 5.009348580999358, + 5.134795276000659, + 5.264739061, + 5.37781595100023, + 5.499105784998392, + 5.620544957000675, + 5.7488538209981925, + 5.87902837799993, + 6.0195663739978045, + 6.1467528530010895, + 6.260794511999848, + 6.39451535599801, + 6.534690880998824, + 6.6660896979992685, + 6.789841363999585, + 6.913929981998081, + 7.061226907000673, + 7.206382092001149, + 7.36402835500121, + 7.5031469499990635, + 7.63544261199786, + 7.763814428999467, + 7.896509950998734, + 8.015378063999378, + 8.152979185000731, + 8.300920040001074, + 8.427796148000198, + 8.559130218000064, + 8.69573959599802, + 8.821535795999807, + 8.938760191998881, + 9.074636565997935, + 9.213950737997948, + 9.354953439000383, + 9.477263633998518, + 9.612464713998634, + 9.74203547900106, + 9.86132764300055, + 9.979825841001002, + 10.12119108999832, + 10.258363234999706, + 10.420643118999578, + 10.577907529997901, + 10.698823839000397, + 10.807552329999453, + 10.914478652997786, + 11.051883387997805, + 11.192668820000108, + 11.316406899997673, + 11.445146433998161, + 11.579534545999195, + 11.68572556000072, + 11.793344065998099, + 11.907194167997659, + 12.042962510000507, + 12.168218549999438, + 12.28161270000055, + 12.411520200999803, + 12.5296655739985, + 12.657397521998064, + 12.791830178000964, + 12.918078891998448, + 13.04175689999829, + 13.165117679000105, + 13.297023207000166, + 13.409357755997917, + 13.521977825999784, + 13.622777656000835, + 13.74612598299791, + 13.865432763999706, + 14.007563357998151, + 14.113029722000647, + 14.218362671999785, + 14.319356891999632, + 14.411220044999936, + 14.49799963699843 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 127.13789099871065, + "p95": 147.29692500259262, + "max": 487.9246000018611 + } + }, + { + "start": 28667.205029691, + "first": 28667.555903202, + "last": 28681.211997295, + "end": 28681.212099631, + "elapsed_s": 14.007069940002111, + "chunk_offsets_s": [ + 0.3508735110008274, + 0.5854607540022698, + 0.7319508450018475, + 0.8746427090009092, + 1.0064265850014635, + 1.1238587820007524, + 1.2325587530031044, + 1.3531047030010086, + 1.4833157230023062, + 1.6011595859999943, + 1.7167626690024917, + 1.8280300030019134, + 1.9408819590025814, + 2.0681922040021163, + 2.2029960210020363, + 2.3438976910001657, + 2.484149180003442, + 2.621207209002023, + 2.737846335003269, + 2.8546179870027117, + 2.985830078003346, + 3.1071506980006234, + 3.2260257480011205, + 3.360698427000898, + 3.4818623820028733, + 3.607185541000945, + 3.7359355370026606, + 3.8520114250022743, + 3.987289494001743, + 4.125694432001183, + 4.25305240400121, + 4.378162223001709, + 4.4927253180030675, + 4.60831454000072, + 4.73371344500265, + 4.878919384002074, + 5.010070296000777, + 5.134435807001864, + 5.263511057000869, + 5.378620794002927, + 5.499249316002533, + 5.620905256000697, + 5.748942259000614, + 5.878856683000777, + 6.019494673000736, + 6.146826821001014, + 6.260809635001351, + 6.394457853002677, + 6.534798923999915, + 6.665673943000002, + 6.790064051001536, + 6.913738220002415, + 7.061172386001999, + 7.206711499002267, + 7.364594698003202, + 7.502762936001091, + 7.635438166002132, + 7.763629851000587, + 7.896439667001687, + 8.01498963800259, + 8.152564863001317, + 8.299765191000915, + 8.42844856100055, + 8.558277252002881, + 8.694565034002153, + 8.821109014003014, + 8.938725407002494, + 9.074567089002812, + 9.21378274900053, + 9.354523178000818, + 9.477196161002212, + 9.612531677001243, + 9.742120907001663, + 9.861414369002887, + 9.979752491002728, + 10.12083722800162, + 10.257945564000693, + 10.420852575000026, + 10.57699604100344, + 10.698650993002957, + 10.807689697001479, + 10.914831166002841, + 11.051859005001461, + 11.193228820000513, + 11.316439666003134, + 11.44402742300008, + 11.57955577300163, + 11.685647822003375, + 11.793172862002393, + 11.907130401999893, + 12.043058596002084, + 12.168356810001569, + 12.281040570000187, + 12.411223292001523, + 12.529770103999908, + 12.65747307300262, + 12.792432441001438, + 12.91886804800015, + 13.041448220003076, + 13.16514375600309, + 13.29706440800146, + 13.40954666000107, + 13.522166795002704, + 13.622230218999903, + 13.746315350002988, + 13.865070453000953, + 14.006967604000238 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 127.64536299982865, + "p95": 147.2003279995988, + "max": 234.5872430014424 + } + }, + { + "start": 28667.205832678, + "first": 28667.473911121, + "last": 28681.846353942, + "end": 28681.8465469, + "elapsed_s": 14.640714222001407, + "chunk_offsets_s": [ + 0.26807844300128636, + 0.5845507020021614, + 0.7315579520000028, + 0.874186445002124, + 1.0057560140012356, + 1.123276676000387, + 1.232446752001124, + 1.3524305310020281, + 1.4825983880000422, + 1.60074922000058, + 1.7166549310022674, + 1.827056750000338, + 1.9404435230026138, + 2.0681028310027614, + 2.2017190160004247, + 2.3429790510017483, + 2.4836300870010746, + 2.6206407040008344, + 2.737616992002586, + 2.853936702002102, + 2.9847715940013586, + 3.1069138390012085, + 3.2254849710006965, + 3.3598141450020194, + 3.48139421200176, + 3.6061589300006744, + 3.7346955920002074, + 3.8517238650019863, + 3.987151471999823, + 4.125607427002251, + 4.251923933999933, + 4.377656786000443, + 4.492502727000101, + 4.607272618999559, + 4.732700586002466, + 4.877753558001132, + 5.009557404002408, + 5.133767139999691, + 5.262907575001009, + 5.377293158002431, + 5.498111413002334, + 5.62000030100171, + 5.74866233099965, + 5.877918215002865, + 6.018357660002948, + 6.146427037001558, + 6.260359059000621, + 6.393529428001784, + 6.5343129739994765, + 6.665256743999635, + 6.789547591000883, + 6.913321743002598, + 7.0600665290003235, + 7.205750469001941, + 7.363657758000045, + 7.502766953999526, + 7.634440728001209, + 7.763208552001743, + 7.895857782001258, + 8.01451933700082, + 8.151537632002146, + 8.299798169002315, + 8.427503732000332, + 8.557876541999576, + 8.694050883001182, + 8.820892724001169, + 8.938108063000982, + 9.073621359999379, + 9.213239774002432, + 9.353868913000042, + 9.47587052200106, + 9.612192374999722, + 9.7420403670003, + 9.860875786001998, + 9.978663334000885, + 10.119963172001007, + 10.258129894002195, + 10.420446611002262, + 10.5772852600021, + 10.697660681002162, + 10.807202514999517, + 10.914397270000336, + 11.051270652002131, + 11.191878790999908, + 11.315798043000541, + 11.443800004002696, + 11.579112630002783, + 11.684552196002187, + 11.792177928000456, + 11.906552274002024, + 12.04248219800138, + 12.167842568000196, + 12.280478001001029, + 12.410032085001149, + 12.529102164000506, + 12.656818974999624, + 12.790469294002833, + 12.917526060002274, + 13.04034324100212, + 13.16481838999971, + 13.296550580002076, + 13.409067710999807, + 13.521632683001371, + 13.622422999000264, + 13.745542961001775, + 13.864608771000348, + 14.006836029002443, + 14.112222822001058, + 14.217640885999572, + 14.318698609000421, + 14.41061695100143, + 14.496967561000929, + 14.57310984400101, + 14.640521264002018 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 127.05676599944127, + "p95": 147.00724999784143, + "max": 316.47225900087506 + } + }, + { + "start": 28667.205184031, + "first": 28667.387831077, + "last": 28680.502334424, + "end": 28680.502774507, + "elapsed_s": 13.297590475998732, + "chunk_offsets_s": [ + 0.18264704600005643, + 0.5852501809968089, + 0.7320817699983309, + 0.8751423959984095, + 1.0064811179981916, + 1.1238437389984028, + 1.2331463269983942, + 1.3530374609981664, + 1.4827568999971845, + 1.6011620149984083, + 1.7168901019977056, + 1.8275645349967817, + 1.9413427079998655, + 2.0682929329996114, + 2.202770309999323, + 2.3433887449973554, + 2.4841648649999115, + 2.621498177999456, + 2.7385439249992487, + 2.85485722699741, + 2.9850544769979024, + 3.107359006997285, + 3.226034688999789, + 3.3604070959991077, + 3.4822062209968863, + 3.607141327000136, + 3.7354077780000807, + 3.8525727950000146, + 3.9868878519991995, + 4.125885576999281, + 4.252963289000036, + 4.378245617997891, + 4.492487997998978, + 4.607401173998369, + 4.733284977999574, + 4.879092754999874, + 5.0091546999974526, + 5.134586774998752, + 5.263798590996885, + 5.37773390700022, + 5.498718544997246, + 5.620110713000031, + 5.749232142999972, + 5.878948022000259, + 6.019422182998824, + 6.146851662997506, + 6.2608457379974425, + 6.394404806997045, + 6.535120459997415, + 6.665333155997359, + 6.790408875000139, + 6.913896373000171, + 7.060878751999553, + 7.206735136998759, + 7.364364126999135, + 7.503082482999162, + 7.635219575997326, + 7.764126709000266, + 7.896622311000101, + 8.015034350999485, + 8.151937709997583, + 8.300108612998883, + 8.428212732997054, + 8.558446522998565, + 8.694908080997266, + 8.82146592799836, + 8.938660970998171, + 9.07406222499776, + 9.214364758998272, + 9.355083192000166, + 9.476409017999686, + 9.612565510997229, + 9.742164901999786, + 9.861850049997884, + 9.979397283997969, + 10.120521369997732, + 10.258385637000174, + 10.420559412999864, + 10.577391683000315, + 10.698365257998375, + 10.80808469099793, + 10.914430171997083, + 11.052198933997715, + 11.192943388999993, + 11.31656024899712, + 11.444512481997663, + 11.579492171000311, + 11.685137565000332, + 11.792767799997819, + 11.907149949998711, + 12.04308907199811, + 12.168388504000177, + 12.281879933998425, + 12.410485387998051, + 12.530072500998358, + 12.658078704000218, + 12.790902561999246, + 12.917980444999557, + 13.041544763000275, + 13.165142149999156, + 13.297150392998446 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 128.1852854990575, + "p95": 146.9823789993825, + "max": 402.6031349967525 + } + }, + { + "start": 28667.20482222, + "first": 28667.651964268, + "last": 28682.124462419, + "end": 28682.124523289, + "elapsed_s": 14.919701069000439, + "chunk_offsets_s": [ + 0.44714204800038715, + 0.5854960210017452, + 0.7328600410000945, + 0.8749754590025987, + 1.0069078270025784, + 1.1243465180014027, + 1.233283608002239, + 1.3534927370019432, + 1.4830481380013225, + 1.6016019449998566, + 1.7174234470003285, + 1.8279777450006804, + 1.940951540000242, + 2.068988172999525, + 2.2029950190008094, + 2.3438085750021855, + 2.484851155000797, + 2.621975170000951, + 2.738567177002551, + 2.855020859002252, + 2.9854825350012106, + 3.107578590999765, + 3.226448598001298, + 3.360987322001165, + 3.482525295999949, + 3.6070291770010954, + 3.7356256540006143, + 3.8524766820009972, + 3.9878454690006038, + 4.126079309000488, + 4.253138623000268, + 4.3787255179995554, + 4.49329370200212, + 4.607895314999041, + 4.733792904000438, + 4.878898592000041, + 5.009781009001017, + 5.13511272300093, + 5.2642123000005085, + 5.378354743999807, + 5.498935814001015, + 5.62082670000018, + 5.749708323000959, + 5.879340395000327, + 6.019496364002407, + 6.147495200999401, + 6.261249613002292, + 6.394729687999643, + 6.535375859999476, + 6.6660388950003835, + 6.790492516000086, + 6.9145199630002026, + 7.060864510000101, + 7.207046267001715, + 7.364355915000488, + 7.503706473999046, + 7.635700332000852, + 7.764272677999543, + 7.897059340000851, + 8.015315540000302, + 8.153271106999455, + 8.300267875001737, + 8.428243205002218, + 8.55863910500193, + 8.695636704000208, + 8.821958699001698, + 8.939229457002511, + 9.074362171002576, + 9.21450014400034, + 9.355567546001112, + 9.477171980001003, + 9.613061062002089, + 9.742786453000008, + 9.862429270000575, + 9.979842521999672, + 10.121421463001752, + 10.25890779600013, + 10.421135363001667, + 10.577366104000248, + 10.69877700500001, + 10.808161257999018, + 10.915362551000726, + 11.05283848900217, + 11.19309553399944, + 11.316762972001015, + 11.444457780999073, + 11.579909880001651, + 11.685380081002222, + 11.793061671000032, + 11.90800101000059, + 12.042884320002486, + 12.168815179000376, + 12.28180715200142, + 12.411321962001239, + 12.530611261001468, + 12.658169573001942, + 12.791782737000176, + 12.918466712002555, + 13.041620603999036, + 13.16560255800141, + 13.297655805999966, + 13.410132962999342, + 13.522698304001096, + 13.623480572001426, + 13.74633188000007, + 13.865896022001834, + 14.008022755002457, + 14.113337554001191, + 14.218908287999511, + 14.31964080599937, + 14.411564531001204, + 14.497890533999453, + 14.574276176001149, + 14.641607633002423, + 14.69262804400205, + 14.749323744999856, + 14.813325687002362, + 14.861945459000708, + 14.919640199001151 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 126.12642699969001, + "p95": 146.34454699989874, + "max": 162.22756700153695 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 86.9562645739488, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 14.75473678599883, + "end_to_end_tps": 86.75180171391649, + "timing_valid": true, + "steady_tps": 91.15448374808312, + "per_stream_tps": [ + 18.0, + 19.0, + 18.0, + 19.7, + 17.4 + ], + "ttft_s": [ + 0.18, + 0.43, + 0.35, + 0.26, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.17984645799879218, + "decode_tps": 18.00825038118337, + "prefill_tps": null, + "itl_mean_ms": 55.5301030823566, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.34326500061434, + "round_gap_p50_ms": 125.80395500117447, + "round_gap_p95_ms": 150.29496900024242, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.43008793400076684, + "decode_tps": 18.998179076163094, + "prefill_tps": null, + "itl_mean_ms": 52.636623541184235, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.57645499997307, + "round_gap_p50_ms": 125.66524800058687, + "round_gap_p95_ms": 145.79212100215955, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3488270369998645, + "decode_tps": 18.047313708136905, + "prefill_tps": null, + "itl_mean_ms": 55.40990843136587, + "itl_p50_ms": 0.0, + "itl_p95_ms": 142.15927899931557, + "round_gap_p50_ms": 124.96785500115948, + "round_gap_p95_ms": 150.36553399841068, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.26350179399742046, + "decode_tps": 19.74772824832611, + "prefill_tps": null, + "itl_mean_ms": 50.63873613334555, + "itl_p50_ms": 0.0, + "itl_p95_ms": 142.55692000006093, + "round_gap_p50_ms": 126.39929600118194, + "round_gap_p95_ms": 149.54939699964598, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.09208107499944163, + "decode_tps": 17.39125291478976, + "prefill_tps": null, + "itl_mean_ms": 57.50017005097927, + "itl_p50_ms": 0.0, + "itl_p95_ms": 142.849349998869, + "round_gap_p50_ms": 124.7120600019116, + "round_gap_p95_ms": 149.99238800010062, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 54.3431082478463, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.34326500061434, + "round_gap_p50_ms": 125.47343599726446, + "round_gap_p95_ms": 149.99238800010062, + "round_gap_mean_ms": 126.43697630657671, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 28682.126672437, + "first": 28682.306518895, + "last": 28696.466695181, + "end": 28696.46696454, + "elapsed_s": 14.340292102999229, + "chunk_offsets_s": [ + 0.17984645799879218, + 0.5547895660020004, + 0.6988919399991573, + 0.8410695749989827, + 0.969542040998931, + 1.0869648859988956, + 1.1992468839998764, + 1.3192126110006939, + 1.4490020869998261, + 1.5675719680002658, + 1.6822587930000736, + 1.7942424040011247, + 1.9084315639993292, + 2.02902481899946, + 2.169272513001488, + 2.3127481049996277, + 2.4511345319988322, + 2.586467443001311, + 2.7009078430019144, + 2.818379847001779, + 2.9446031379993656, + 3.0616800200004946, + 3.1863219260012556, + 3.32112919499923, + 3.441382461001922, + 3.5661431980006455, + 3.692738403999101, + 3.8101866280012473, + 3.94251469899973, + 4.07756991199858, + 4.21074505100114, + 4.3319640410009015, + 4.445779298999696, + 4.560864702998515, + 4.684750317999715, + 4.828925343001174, + 4.954945585999667, + 5.081119508999109, + 5.215379715999006, + 5.32986419000008, + 5.449709216001793, + 5.57038911500058, + 5.696193070001755, + 5.82868316999884, + 5.96520856600182, + 6.092231916998571, + 6.209611705999123, + 6.34225967599923, + 6.47928247299933, + 6.607953659000486, + 6.73135953900055, + 6.856680890999996, + 6.9957409179987735, + 7.146035886999016, + 7.304850769000041, + 7.443082264999248, + 7.577321510998445, + 7.704625744001532, + 7.83951259999958, + 7.953107304998412, + 8.090725899000972, + 8.24166890700144, + 8.367751597001188, + 8.500436827001977, + 8.635841322000488, + 8.764271218999056, + 8.880601444001513, + 9.009048021998751, + 9.155033621998882, + 9.296376886999496, + 9.419187431001774, + 9.554595845998847, + 9.68156992600052, + 9.803042856001412, + 9.919511316998978, + 10.053986475999409, + 10.19743342700167, + 10.357812851001654, + 10.515133600001718, + 10.63771822900162, + 10.744935046001046, + 10.850296913999046, + 10.979452742998546, + 11.117290859001514, + 11.232900610000797, + 11.357483829000557, + 11.49292125500142, + 11.596442828998988, + 11.70308085799843, + 11.814568793000944, + 11.942547124999692, + 12.061916967999423, + 12.181659533998754, + 12.307610305000708, + 12.424537661001523, + 12.548859341000934, + 12.680721631000779, + 12.806573518999357, + 12.921380042000237, + 13.038234850999288, + 13.177964995000366, + 13.288871725999343, + 13.40005606900013, + 13.499636304000887, + 13.622301545001392, + 13.739702203001798, + 13.854918356999406, + 13.956411877999926, + 14.058194861001539, + 14.164490500999818, + 14.253173616998538, + 14.340022743999725 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 125.80395500117447, + "p95": 150.29496900024242, + "max": 374.9431080032082 + } + }, + { + "start": 28682.128857579, + "first": 28682.558945513, + "last": 28695.981284516, + "end": 28695.9820512, + "elapsed_s": 13.853193621001992, + "chunk_offsets_s": [ + 0.43008793400076684, + 0.5523542959999759, + 0.6965433910008869, + 0.8387908610020531, + 0.967634396001813, + 1.084419198003161, + 1.1972280320005666, + 1.3177135470032226, + 1.4465528380023898, + 1.5656669020027039, + 1.680032554002537, + 1.7922573980031302, + 1.9061417730008543, + 2.027222699001868, + 2.167799154001841, + 2.3101170290028676, + 2.448802730999887, + 2.584371959001146, + 2.6987716750008985, + 2.81690079100008, + 2.9422194580001815, + 3.0599103560016374, + 3.18450661800307, + 3.3184928780028713, + 3.4387927160023537, + 3.5644579640029406, + 3.690953792000073, + 3.80783359400084, + 3.9402749490000133, + 4.075634751003236, + 4.2082869639998535, + 4.329522031999659, + 4.443472498001938, + 4.558714263002912, + 4.683369431000756, + 4.826059248000092, + 4.953151746001822, + 5.079200317002687, + 5.2134352550019685, + 5.326963090003119, + 5.447829961001844, + 5.568006416000571, + 5.693941043002269, + 5.826586819002841, + 5.962975738002569, + 6.089853422003216, + 6.207244578999962, + 6.339562320001278, + 6.477535805999651, + 6.605655900002603, + 6.729364285001793, + 6.854067666001356, + 6.993515954000031, + 7.143590324001707, + 7.302236118001019, + 7.440337580002961, + 7.574933800002327, + 7.702039960000548, + 7.8365472450022935, + 7.950968917000864, + 8.088273618999665, + 8.239292218000628, + 8.365333615001873, + 8.497707339000044, + 8.633549830999982, + 8.761639653999737, + 8.8785829940025, + 9.006717850999848, + 9.152509972002008, + 9.294292984002823, + 9.417449432003195, + 9.552224478000426, + 9.679295917001582, + 9.80067182300263, + 9.91758460300116, + 10.052007907001098, + 10.195391646000644, + 10.356325128002936, + 10.51250985000297, + 10.635092800002894, + 10.742013214003236, + 10.848501045002195, + 10.977769061002618, + 11.114861092002684, + 11.230171349001466, + 11.354611586000829, + 11.491297280001163, + 11.59405956900082, + 11.700219100999675, + 11.812524447999749, + 11.94076193300134, + 12.059577644002275, + 12.1800260050004, + 12.305500592003227, + 12.422098751001613, + 12.546915920000174, + 12.67825718899985, + 12.804229951001616, + 12.919105487002525, + 13.036007156002597, + 13.17557060600302, + 13.28707132800264, + 13.39762005800003, + 13.496999771999981, + 13.61943931100177, + 13.73703386499983, + 13.852426937002747 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 125.79993750114227, + "p95": 145.79212100215955, + "max": 160.9334820022923 + } + }, + { + "start": 28682.127988257, + "first": 28682.476815294, + "last": 28696.606341944, + "end": 28696.606415562, + "elapsed_s": 14.478427304999059, + "chunk_offsets_s": [ + 0.3488270369998645, + 0.5529135010001482, + 0.6973495259990159, + 0.8395088049983315, + 0.9685681869996188, + 1.0857511759968475, + 1.1982417479994183, + 1.3183597030001692, + 1.4474940769978275, + 1.566913251997903, + 1.6813401209983567, + 1.793332269997336, + 1.9068498619999446, + 2.027633370998956, + 2.1684630229974573, + 2.3107622869974875, + 2.4496172459985246, + 2.585457448996749, + 2.6995051249978133, + 2.816859979997389, + 2.9431318609968002, + 3.060953920998145, + 3.1849328849966696, + 3.319224150996888, + 3.439597128999594, + 3.5650062809982046, + 3.69177588799721, + 3.809001931997045, + 3.9413566019975406, + 4.076428901997133, + 4.209220582997659, + 4.330946355999913, + 4.444282712000131, + 4.559419776996947, + 4.683893024997815, + 4.826863795999088, + 4.953944957000203, + 5.080215920999763, + 5.21424394999849, + 5.328302291996806, + 5.4492120949980745, + 5.569263421999494, + 5.694644696999603, + 5.827424756000255, + 5.964085393999994, + 6.090763992997381, + 6.20805246099917, + 6.340675450999697, + 6.4781744979991345, + 6.606380882996746, + 6.730483218998415, + 6.855173902997194, + 6.994210455999564, + 7.144575989997975, + 7.30315131399766, + 7.441598637997231, + 7.5756563549985, + 7.703068101000099, + 7.838110188997234, + 7.952048662999005, + 8.08977477499866, + 8.240252149997104, + 8.365820772000006, + 8.498815345999901, + 8.634198913998262, + 8.762752296999679, + 8.879808787998627, + 9.007856223997806, + 9.153127406996646, + 9.295600754998304, + 9.41799459199683, + 9.552910569000232, + 9.680118549997133, + 9.801944644998002, + 9.91906135699901, + 10.052833594996628, + 10.196559301999514, + 10.356927625998651, + 10.51343889599957, + 10.63617350899949, + 10.742976464000094, + 10.8494295399978, + 10.97846906599807, + 11.116130999998859, + 11.231880228999216, + 11.35570391299916, + 11.49187416299901, + 11.595207832997403, + 11.70088647999728, + 11.813553746997059, + 11.94116481999663, + 12.061194395999337, + 12.181114119997801, + 12.30608197499896, + 12.422718965997774, + 12.547055070997885, + 12.678978971998731, + 12.804926899996644, + 12.919771746997867, + 13.036740866999025, + 13.17691855099838, + 13.287377914999524, + 13.398676742999669, + 13.49803345199689, + 13.620782852998673, + 13.738172007997491, + 13.85336984499736, + 13.954409709000174, + 14.056951172999106, + 14.16291055399779, + 14.251696762999927, + 14.33881234099681, + 14.413401254998462, + 14.478353686998162 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 124.96785500115948, + "p95": 150.36553399841068, + "max": 204.08646400028374 + } + }, + { + "start": 28682.12834699, + "first": 28682.391848784, + "last": 28695.304726498, + "end": 28695.305099204, + "elapsed_s": 13.176752213999862, + "chunk_offsets_s": [ + 0.26350179399742046, + 0.5522003910009516, + 0.6967955599975539, + 0.8393524799976149, + 0.967608154998743, + 1.0844500419989345, + 1.1974730399997497, + 1.3177578129980247, + 1.4468265659997996, + 1.5664876470000308, + 1.680828852000559, + 1.792461328997888, + 1.9068702279982972, + 2.0271162559984077, + 2.167881650999334, + 2.31115278300058, + 2.4489651799995045, + 2.5847098949998326, + 2.6995893189996423, + 2.8172169899989967, + 2.9432031779979297, + 3.0601616780004406, + 3.185635113997705, + 3.3192110060008417, + 3.4401845459979086, + 3.5651800310006365, + 3.6916885469981935, + 3.808435238999664, + 3.9409600279977894, + 4.075965181000356, + 4.208220339998661, + 4.3299545839981874, + 4.443450168997515, + 4.558529294998152, + 4.682936624998547, + 4.82702905199767, + 4.953428347998852, + 5.079613400997914, + 5.213821499997721, + 5.328329080999538, + 5.448567186998844, + 5.568138288999762, + 5.694606377997843, + 5.826833213999635, + 5.963600642000529, + 6.090187326997693, + 6.2073886539983505, + 6.339475634998962, + 6.477012503997685, + 6.605610351998621, + 6.729957560000912, + 6.855247800998768, + 6.993776018000062, + 7.143325414999708, + 7.303118235999136, + 7.441931900000782, + 7.57593994799754, + 7.702843768998719, + 7.836893346997385, + 7.951032413999201, + 8.088193363997561, + 8.240141468999354, + 8.365221864998603, + 8.497813474998111, + 8.633732374997635, + 8.761955322999711, + 8.879565975999867, + 9.007702994000283, + 9.152245925997704, + 9.295418304998748, + 9.417394395000883, + 9.552182010000251, + 9.679575046000537, + 9.800832719000027, + 9.91844616399976, + 10.0522160349974, + 10.196062517999962, + 10.355864548000682, + 10.513411302999884, + 10.635391730000265, + 10.743004883999674, + 10.84818662500038, + 10.977574525997625, + 11.115910488999361, + 11.230388918000244, + 11.3555486869991, + 11.490762509998603, + 11.594348645998252, + 11.70096936399932, + 11.812602473997686, + 11.940298928999255, + 12.060462435998488, + 12.180212308998307, + 12.305509881000035, + 12.422209845997713, + 12.547011709000799, + 12.678448385999218, + 12.804380551999202, + 12.919905205999385, + 13.036297012000432, + 13.176379508000537 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 126.43369249963143, + "p95": 149.54939699964598, + "max": 288.6985970035312 + } + }, + { + "start": 28682.126577608, + "first": 28682.218658683, + "last": 28696.881202046, + "end": 28696.881314394, + "elapsed_s": 14.75473678599883, + "chunk_offsets_s": [ + 0.09208107499944163, + 0.5537781769999128, + 0.698058639998635, + 0.8410175429999072, + 0.9698260650002339, + 1.085967218001315, + 1.1989475789996504, + 1.3191052229994966, + 1.4483324339998944, + 1.567517758001486, + 1.6826757679991715, + 1.7939904390004813, + 1.9083496240018576, + 2.029278157999215, + 2.169441175999964, + 2.312290525998833, + 2.450457005001226, + 2.587376382001821, + 2.700799713999004, + 2.8187040429984336, + 2.944404683999892, + 3.0620029989986506, + 3.1870257730006415, + 3.320471222999913, + 3.441247282000404, + 3.5666761329994188, + 3.6927152889984427, + 3.8099389630006044, + 3.9426535029997467, + 4.0774506160014425, + 4.210079016000236, + 4.331412412000645, + 4.446194683001522, + 4.561061300999427, + 4.685578623000765, + 4.828585521001514, + 4.955099472001166, + 5.081134783999005, + 5.215873069999361, + 5.329034151000087, + 5.449918892998539, + 5.569852697000897, + 5.696552140001586, + 5.828533422001783, + 5.965422689001571, + 6.091874675999861, + 6.209078533000138, + 6.341506296001171, + 6.479023582000082, + 6.607592045998899, + 6.732166186000541, + 6.857074890998774, + 6.99573378200148, + 7.145726170001581, + 7.304772201998276, + 7.443375515998923, + 7.57695817000058, + 7.704030232998775, + 7.838416343001882, + 7.95301819299857, + 8.09019358599835, + 8.24137680000058, + 8.36747085800016, + 8.499333845000365, + 8.635214989000815, + 8.76408042500043, + 8.88145712199912, + 9.009532306001347, + 9.154249856001115, + 9.297464215000218, + 9.418899144999159, + 9.554134816000442, + 9.681400106001092, + 9.802810369001236, + 9.920145392999984, + 10.05415572900165, + 10.197761971001455, + 10.358001169999625, + 10.515090426000825, + 10.637019910998788, + 10.744707975001802, + 10.85007128900179, + 10.979670300999715, + 11.117003218001628, + 11.232681945999502, + 11.357394006001414, + 11.492603131999203, + 11.596006170999317, + 11.702968451998458, + 11.814325102001021, + 11.942340502999286, + 12.062404991000221, + 12.181532671998866, + 12.306941358001495, + 12.423686453999835, + 12.548246736998408, + 12.679850789998454, + 12.805965190000279, + 12.920808349001163, + 13.038459185001557, + 13.178249632001098, + 13.288320955001836, + 13.399674279000465, + 13.499045030999696, + 13.62181071500163, + 13.739198683000723, + 13.85493768699962, + 13.956037718999141, + 14.058163836998574, + 14.164526600001409, + 14.252904338001827, + 14.339819338998495, + 14.414523515999463, + 14.479568705999554, + 14.52903247499853, + 14.584574639000493, + 14.647826489999716, + 14.69663358699836, + 14.754624437999155 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 124.81038250007259, + "p95": 149.99238800010062, + "max": 461.6971020004712 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 87.06851859959646, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 14.730955462000566, + "end_to_end_tps": 86.89185187626433, + "timing_valid": true, + "steady_tps": 91.70820209696785, + "per_stream_tps": [ + 18.4, + 18.9, + 18.0, + 19.7, + 17.4 + ], + "ttft_s": [ + 0.43, + 0.34, + 0.25, + 0.17, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.42782242500106804, + "decode_tps": 18.37587726057092, + "prefill_tps": null, + "itl_mean_ms": 54.41917062352706, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.72740999836242, + "round_gap_p50_ms": 125.44890099889017, + "round_gap_p95_ms": 145.02074100164464, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3387435490003554, + "decode_tps": 18.918122020245782, + "prefill_tps": null, + "itl_mean_ms": 52.85936938824164, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.9572579980013, + "round_gap_p50_ms": 125.85369999942486, + "round_gap_p95_ms": 144.89657000012812, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2531475949981541, + "decode_tps": 17.958352941302135, + "prefill_tps": null, + "itl_mean_ms": 55.684393956870935, + "itl_p50_ms": 0.0, + "itl_p95_ms": 142.05755999864778, + "round_gap_p50_ms": 125.56197099911515, + "round_gap_p95_ms": 145.58781800224097, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17139017099907505, + "decode_tps": 19.676957265702438, + "prefill_tps": null, + "itl_mean_ms": 50.820865568632996, + "itl_p50_ms": 0.0, + "itl_p95_ms": 142.18782500029192, + "round_gap_p50_ms": 126.6261230011878, + "round_gap_p95_ms": 145.0621959993441, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08722157700321986, + "decode_tps": 17.41370371991929, + "prefill_tps": null, + "itl_mean_ms": 57.42603733725606, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.9231479994778, + "round_gap_p50_ms": 125.05066500307294, + "round_gap_p95_ms": 145.09513499797322, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 54.24196737490574, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.33573700019042, + "round_gap_p50_ms": 125.6391220013029, + "round_gap_p95_ms": 145.0621959993441, + "round_gap_mean_ms": 126.20165766971682, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 28696.883912414, + "first": 28697.311734839, + "last": 28711.188623348, + "end": 28711.188762083, + "elapsed_s": 14.30484966899894, + "chunk_offsets_s": [ + 0.42782242500106804, + 0.5509237709993613, + 0.6922595079995517, + 0.8351958419989387, + 0.9619478210006491, + 1.0759048639993125, + 1.183186599999317, + 1.302102976002061, + 1.436704046001978, + 1.5528920760007168, + 1.6670243080006912, + 1.7751650360005442, + 1.8898676270000578, + 2.008509978000802, + 2.1419989640016865, + 2.282726374000049, + 2.4277471150016936, + 2.566626235999138, + 2.6815991629991913, + 2.7985368379995634, + 2.9250364959989383, + 3.0401320259989006, + 3.158676466999168, + 3.2887872380015324, + 3.4179786690001492, + 3.5400752249988727, + 3.6664878129995486, + 3.7819504600010987, + 3.913536041000043, + 4.044721032001689, + 4.169609302000026, + 4.291366776000359, + 4.413691671001288, + 4.5275463969992416, + 4.6511965339996095, + 4.794529610000609, + 4.91926412400062, + 5.041326717000629, + 5.166792969001108, + 5.279120549999789, + 5.4072377779993985, + 5.5270588169987604, + 5.654084857000271, + 5.783103682999354, + 5.919303081998805, + 6.041440330001933, + 6.1536883160006255, + 6.283614070998738, + 6.430257332998735, + 6.558142139001575, + 6.682352986001206, + 6.808170082000288, + 6.947939474001032, + 7.090838385000097, + 7.2474131870003475, + 7.390711610001745, + 7.522923463999177, + 7.648267562999536, + 7.780303847001051, + 7.892959869001061, + 8.027886061001482, + 8.17345780100004, + 8.29890670199893, + 8.439507195998885, + 8.5724324469993, + 8.700214794000203, + 8.817967928000144, + 8.944964586000424, + 9.083755204999761, + 9.223751954999898, + 9.349262722000276, + 9.486601773001894, + 9.613566054002149, + 9.73200664199976, + 9.847876211999392, + 9.983251810001093, + 10.118000261001725, + 10.276882217000093, + 10.440862981002283, + 10.562384360000578, + 10.668917270999373, + 10.778011062000587, + 10.910948434000602, + 11.046969503000582, + 11.159782299000653, + 11.286767699999473, + 11.430440052001359, + 11.535257925999758, + 11.64215156100181, + 11.755303894999088, + 11.886307475000649, + 12.008991380000225, + 12.121013947999018, + 12.246967686998687, + 12.370400177998818, + 12.499591244999465, + 12.63307663500018, + 12.763412296000752, + 12.879699583001639, + 12.999988075000147, + 13.130513321000763, + 13.240388720001647, + 13.35754307300158, + 13.460995810000895, + 13.582711844999722, + 13.70125626600202, + 13.817939101001684, + 13.921982867999759, + 14.026659688999644, + 14.125482136001665, + 14.21330712100098, + 14.304710934000468 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 125.44890099889017, + "p95": 145.02074100164464, + "max": 163.98076400218997 + } + }, + { + "start": 28696.884287651, + "first": 28697.2230312, + "last": 28710.702170394, + "end": 28710.702325332, + "elapsed_s": 13.818037681001442, + "chunk_offsets_s": [ + 0.3387435490003554, + 0.5500420990028942, + 0.6919993570008955, + 0.8352679859999625, + 0.9619766440009698, + 1.0753862759993353, + 1.1830364910019853, + 1.3014584720003768, + 1.4364072440002928, + 1.5527643929999613, + 1.6669628250019741, + 1.775111367001955, + 1.8901484260022698, + 2.0084702700005437, + 2.1412557830008154, + 2.2824116200026765, + 2.4273081900028046, + 2.5655814010024187, + 2.6810000490004313, + 2.7980587060010293, + 2.924434286000178, + 3.040004217000387, + 3.158718424001563, + 3.288251511999988, + 3.4171237900009146, + 3.5400747759995284, + 3.6669643220011494, + 3.7824681510028313, + 3.91385297600209, + 4.0441227270021045, + 4.1695352450005885, + 4.290624550001667, + 4.413409902001149, + 4.527399028000218, + 4.6503866400016705, + 4.793820893002703, + 4.919417555000109, + 5.040812429000653, + 5.166251511000155, + 5.278876617001515, + 5.407008448000852, + 5.526423485000123, + 5.65313122400039, + 5.782844510002178, + 5.919394000000466, + 6.041518560999975, + 6.153710099999444, + 6.283556120000867, + 6.429485138000018, + 6.558026245002111, + 6.681321575000766, + 6.807520427999407, + 6.94738957500158, + 7.090696488001413, + 7.246823246001441, + 7.390181173999736, + 7.522707822001394, + 7.64769374000025, + 7.780024605002836, + 7.892476993001765, + 8.027942123000685, + 8.172780630000489, + 8.298634329999913, + 8.439052886002173, + 8.571971397002926, + 8.700071676001244, + 8.818146854002407, + 8.945289250001224, + 9.082795604001149, + 9.223519453000335, + 9.349158575001638, + 9.486398574001214, + 9.6134393320026, + 9.731762120001804, + 9.84780081000281, + 9.98309896399951, + 10.117986872002803, + 10.276828239002498, + 10.440353821999452, + 10.56153370299944, + 10.668650565999997, + 10.777780944001279, + 10.910692710000149, + 11.046929210002418, + 11.159687983999902, + 11.286202926999977, + 11.430347077999613, + 11.535264107002149, + 11.642165576002299, + 11.755121927999426, + 11.885543624001002, + 12.008983407002233, + 12.12152149800022, + 12.246734947999357, + 12.370318714001769, + 12.4988744220027, + 12.63277653300247, + 12.762474376999307, + 12.87983605500267, + 12.99936813900058, + 13.130267545002425, + 13.240172890000395, + 13.35692781400212, + 13.460257029000786, + 13.582673381999484, + 13.701060758001404, + 13.817882743001974 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 126.02627649903297, + "p95": 144.89657000012812, + "max": 211.29855000253883 + } + }, + { + "start": 28696.884549669, + "first": 28697.137697264, + "last": 28711.337217723, + "end": 28711.337544099, + "elapsed_s": 14.452994429997489, + "chunk_offsets_s": [ + 0.2531475949981541, + 0.5498554699988745, + 0.6919130299975222, + 0.8352345889979915, + 0.9617870379988744, + 1.0757540329977928, + 1.18261074199836, + 1.3013564129978477, + 1.4358150209991436, + 1.5524238299985882, + 1.6665524009986257, + 1.7747653829974297, + 1.8892805959985708, + 2.0079316989977087, + 2.1417605610004102, + 2.2822891319992777, + 2.4267881189989566, + 2.5661498039989965, + 2.6811239359994943, + 2.7972815510001965, + 2.9246590889997606, + 3.039419567998266, + 3.1577212029988004, + 3.288322161999531, + 3.417507731999649, + 3.5396822379989317, + 3.666123365997919, + 3.7815905709976505, + 3.9135131949988136, + 4.043174530997931, + 4.169428020999476, + 4.290607916998852, + 4.4132385379998595, + 4.526966049998009, + 4.650500766998448, + 4.793747717998485, + 4.919097283000156, + 5.040741592998529, + 5.166112651997537, + 5.2785692929974175, + 5.406956048998836, + 5.527585448999162, + 5.653147419998277, + 5.7823723959991185, + 5.918610159998934, + 6.0415993469978275, + 6.153541936997499, + 6.283364047998475, + 6.429437430000689, + 6.557626436999271, + 6.681540006997238, + 6.807744399000512, + 6.9475113169974065, + 7.090375170999323, + 7.246720137998636, + 7.3898070420000295, + 7.522098517998529, + 7.647539615998539, + 7.779892127000494, + 7.892576754999027, + 8.02716923999833, + 8.17275705800057, + 8.298993420998158, + 8.438538861999405, + 8.57152941700042, + 8.699977938998927, + 8.817400725998596, + 8.944271802000003, + 9.082785177997721, + 9.222862214999623, + 9.348480595999717, + 9.485795016000338, + 9.612857312000415, + 9.731312503998197, + 9.847164312999666, + 9.982444204997591, + 10.117302112998004, + 10.276173638998443, + 10.439712586998212, + 10.561685860997386, + 10.668188931998884, + 10.777291350997984, + 10.910500252997736, + 11.046252236999862, + 11.159081817000697, + 11.286044004998985, + 11.429877415997908, + 11.534551573000499, + 11.641413123998063, + 11.754718411997601, + 11.885118377998879, + 12.008288294000522, + 12.120431639999879, + 12.246159528000135, + 12.369903776998399, + 12.498899633999827, + 12.632158022999647, + 12.762609418998181, + 12.879131462999794, + 12.999602287000016, + 13.129814160998649, + 13.240159528999357, + 13.356831540997518, + 13.460198092998326, + 13.582009988000209, + 13.700976919997629, + 13.817514954000217, + 13.921237948998169, + 14.025956363999285, + 14.125050021997595, + 14.212616106000496, + 14.304185606000829, + 14.385158215998672, + 14.452668054000242 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 125.56197099911515, + "p95": 145.58781800224097, + "max": 296.7078750007204 + } + }, + { + "start": 28696.883979241, + "first": 28697.055369412, + "last": 28710.014690132, + "end": 28710.014878066, + "elapsed_s": 13.13089882499844, + "chunk_offsets_s": [ + 0.17139017099907505, + 0.5499353390005126, + 0.6921231640008045, + 0.8353165469998203, + 0.9619625199993607, + 1.075935742999718, + 1.1832380980013113, + 1.3022733700017852, + 1.436570607998874, + 1.5531163240011665, + 1.66707184400002, + 1.77561980199971, + 1.890558774000965, + 2.0086621400005242, + 2.142114570000558, + 2.2823994159989525, + 2.427290098999947, + 2.566201492998516, + 2.6813988190006057, + 2.7979880339989904, + 2.9245552069987752, + 3.040426693001791, + 3.1586774889983644, + 3.288671255999361, + 3.4172537790000206, + 3.540128022999852, + 3.6667541460010398, + 3.7825801619983395, + 3.9135677209997084, + 4.043940971001575, + 4.169270606998907, + 4.290625342000567, + 4.413358270001481, + 4.527983264000795, + 4.650967808000132, + 4.794038441999874, + 4.919862996001029, + 5.0409683139987465, + 5.166925945999537, + 5.279743442999461, + 5.407234201000392, + 5.526921518001473, + 5.653808744998969, + 5.783262127999478, + 5.919447126001614, + 6.041478612001811, + 6.153766378, + 6.284048277000693, + 6.429920864000451, + 6.558400174999406, + 6.681730652999249, + 6.807930874998419, + 6.948275473001559, + 7.091296007001802, + 7.247214638999139, + 7.390577234000375, + 7.52292049099924, + 7.648294246999285, + 7.780493772999762, + 7.893219033001515, + 8.02819541200006, + 8.173257607999403, + 8.298696072000894, + 8.43896715899973, + 8.572198805999506, + 8.700043525001092, + 8.818205392999516, + 8.945443191001687, + 9.08360866500152, + 9.224143177998485, + 9.349619605000044, + 9.486287291998451, + 9.613675601998693, + 9.732196174001729, + 9.84794531299849, + 9.983478026999364, + 10.118147181001405, + 10.277272134000668, + 10.440489729000547, + 10.562204647001636, + 10.668672039999365, + 10.778044616999978, + 10.911474291999184, + 11.047172493999824, + 11.159855311001593, + 11.287008726998465, + 11.430569621999894, + 11.535335788001248, + 11.642183347001264, + 11.755158684998605, + 11.88645433900092, + 12.009050341999682, + 12.121558317998279, + 12.247367692998523, + 12.37043129200174, + 12.49939858200014, + 12.632813783999154, + 12.76326048700139, + 12.88007178800035, + 13.00036321800144, + 13.13071089100049 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 126.6360480003641, + "p95": 145.0621959993441, + "max": 378.54516800143756 + } + }, + { + "start": 28696.883794796, + "first": 28696.971016373, + "last": 28711.614655894, + "end": 28711.614750258, + "elapsed_s": 14.730955462000566, + "chunk_offsets_s": [ + 0.08722157700321986, + 0.550289540002268, + 0.6922126880017458, + 0.8356005980022019, + 0.9624197340017417, + 1.0762384020017635, + 1.1835749270030647, + 1.302034991000255, + 1.4366975540033309, + 1.5531101420019695, + 1.6674062330021115, + 1.7758830770035274, + 1.8903653090019361, + 2.0089175880020775, + 2.141829589003464, + 2.2827642940028454, + 2.4277125420012453, + 2.566002572002617, + 2.6816446200027713, + 2.798344964001444, + 2.9249711980010034, + 3.040556853000453, + 3.1585667640028987, + 3.2886881600024935, + 3.417920233001496, + 3.54062288500063, + 3.667229998001858, + 3.7825658090005163, + 3.91407037099998, + 4.044269513000472, + 4.170080254003551, + 4.290895784000895, + 4.413278683001408, + 4.528000134003378, + 4.650799300001381, + 4.794383451000613, + 4.919602353002119, + 5.041368025002157, + 5.166668962003314, + 5.2797830940035055, + 5.407732000003307, + 5.527351645003364, + 5.653786721002689, + 5.783648651002295, + 5.919571781003469, + 6.041836345000775, + 6.15434354700119, + 6.283596383000258, + 6.429904768003325, + 6.558462780001719, + 6.6817153219999454, + 6.8079669600010675, + 6.947774425003445, + 7.091068786001415, + 7.24706074900314, + 7.390781143003551, + 7.522967687000346, + 7.648018352003419, + 7.780565976001526, + 7.892915949003509, + 8.028234364002856, + 8.173329499000829, + 8.299356802002876, + 8.438865957003145, + 8.572500984002545, + 8.700476184003492, + 8.818683628000144, + 8.945363768001698, + 9.083165078001912, + 9.224057745002938, + 9.34951569400073, + 9.486821855000017, + 9.61377220200302, + 9.732319112001278, + 9.848172229001648, + 9.983540400000493, + 10.11840716000006, + 10.277178319000086, + 10.440898351000214, + 10.56214008200186, + 10.669211922002432, + 10.778460847002862, + 10.911977503001253, + 11.047642234003433, + 11.160254899001302, + 11.28663217200301, + 11.430471948002378, + 11.535650149002322, + 11.642612499003008, + 11.755723240003135, + 11.886829374001536, + 12.009614196002076, + 12.121530668002379, + 12.247286165002151, + 12.370899312001711, + 12.499054848001833, + 12.633316004001244, + 12.763255372003186, + 12.87972610700308, + 12.999800255001901, + 13.131046219001291, + 13.24069901200346, + 13.35733113200331, + 13.460809937001613, + 13.583087651000824, + 13.701115505002235, + 13.818508779000695, + 13.922244378001778, + 14.026937310001813, + 14.1259116800029, + 14.21353831000306, + 14.304550540000491, + 14.385839866001334, + 14.4533247640029, + 14.503728144001798, + 14.561666507001064, + 14.625322361000144, + 14.672314579001977, + 14.730861098003516 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 125.13478350228979, + "p95": 145.09513499797322, + "max": 463.0679629990482 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.8, + "per_stream_tps": 66.8, + "ttft_s_max": 0.1, + "request": { + "ttft_s": 0.096, + "decode_tps": 66.785, + "prefill_tps": 145.467, + "itl_mean_ms": 14.973, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.77, + "round_gap_p50_ms": 74.009, + "round_gap_p95_ms": 82.629, + "tokens_per_chunk": 4.571 + }, + "pooled": { + "aggregate_prefill_tps": 145.467, + "itl_mean_ms": 14.973, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.77, + "round_gap_p50_ms": 74.009, + "round_gap_p95_ms": 82.629, + "tokens_per_chunk": 4.571 + }, + "timing_valid": true, + "end_to_end_tps": 65.39729834465851, + "makespan_s": 3.9145347970006696, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.60091218678184, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.9264483120023215, + "end_to_end_tps": 65.19887176852988, + "timing_valid": true, + "per_stream_tps": [ + 66.6 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09759993600164307, + "decode_tps": 66.60091218678184, + "prefill_tps": 143.44271700920288, + "itl_mean_ms": 15.014809364705192, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.25087000310305, + "round_gap_p50_ms": 74.03332600006252, + "round_gap_p95_ms": 82.97204400150804, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 15.014809364705192, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.25087000310305, + "round_gap_p50_ms": 74.03332600006252, + "round_gap_p95_ms": 82.97204400150804, + "round_gap_mean_ms": 69.61411614545135, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 143.44271700920288 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28726.063239449, + "first": 28726.160839385, + "last": 28729.989615773, + "end": 28729.989687761, + "elapsed_s": 3.9264483120023215, + "chunk_offsets_s": [ + 0.09759993600164307, + 0.1534757770023134, + 0.21241270800237544, + 0.26534100100252545, + 0.31939327600048273, + 0.38799113300046884, + 0.46732194400101434, + 0.5408008300000802, + 0.6148341560001427, + 0.6894848620031553, + 0.763981321000756, + 0.8357859610005107, + 0.9122125250032695, + 0.9895804160005355, + 1.0662416619998112, + 1.1444925320029142, + 1.2281963920031558, + 1.3078889349999372, + 1.3815934209997067, + 1.4383316910025314, + 1.4960925770028553, + 1.558528635003313, + 1.6176451440005621, + 1.6958153240011598, + 1.7602601090002281, + 1.8101047900017875, + 1.8693134260029183, + 1.9462850470008561, + 2.025032410001586, + 2.090319115002785, + 2.141611596001894, + 2.21007451100013, + 2.293046555001638, + 2.3723826940004074, + 2.453484492001735, + 2.534747626999888, + 2.6136482380024972, + 2.6890098300027603, + 2.766016577003029, + 2.840256825002143, + 2.9171364800022275, + 2.9970298260013806, + 3.072587423001096, + 3.149496922000253, + 3.2334082820016192, + 3.3059524609998334, + 3.3591994890011847, + 3.424782560003223, + 3.5033811520006566, + 3.5700614270026563, + 3.632373792999715, + 3.706826577999891, + 3.7713623520030524, + 3.8187932140026533, + 3.8684311380020517, + 3.926376324001467 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 74.03332600006252, + "p95": 82.97204400150804, + "max": 83.91136000136612 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.78541485333007, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.9145347970006696, + "end_to_end_tps": 65.39729834465851, + "timing_valid": true, + "per_stream_tps": [ + 66.8 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09624161900137551, + "decode_tps": 66.78541485333007, + "prefill_tps": 145.46721205718606, + "itl_mean_ms": 14.973329164700665, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.77030800207285, + "round_gap_p50_ms": 73.61074299842585, + "round_gap_p95_ms": 82.62877599918284, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.973329164700665, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.77030800207285, + "round_gap_p50_ms": 73.61074299842585, + "round_gap_p95_ms": 82.62877599918284, + "round_gap_mean_ms": 69.42179885452127, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 145.46721205718606 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28729.990599604, + "first": 28730.086841223, + "last": 28733.90504016, + "end": 28733.905134401, + "elapsed_s": 3.9145347970006696, + "chunk_offsets_s": [ + 0.09624161900137551, + 0.15132543499930762, + 0.20493202500074403, + 0.2609051860017644, + 0.31827180099935504, + 0.3877993329988385, + 0.46690634799961117, + 0.5401740090019302, + 0.6126302599986957, + 0.6881150519984658, + 0.762520249998488, + 0.8353472700000566, + 0.9115270719994442, + 0.9904135950018826, + 1.0661348159992485, + 1.143465485998604, + 1.2215463479988102, + 1.3058111509999435, + 1.380888709998544, + 1.436093347001588, + 1.4947118749987567, + 1.554497123001056, + 1.6153153129998827, + 1.6927815700000792, + 1.7542302330002713, + 1.8038506170014443, + 1.865544721000333, + 1.94170090300031, + 2.018715666999924, + 2.0844893290013715, + 2.135156005999306, + 2.197615514000063, + 2.2808954220017768, + 2.36239559599926, + 2.4421962480009825, + 2.5248250240001653, + 2.6059286349991453, + 2.6816006080007355, + 2.7579118290013866, + 2.832778848001908, + 2.910153924000042, + 2.98899966099998, + 3.065687240999978, + 3.1434575490020507, + 3.2212534089994733, + 3.2946455029996287, + 3.3508480970012897, + 3.4159082369988027, + 3.494697793001251, + 3.560039351999876, + 3.6239248480014794, + 3.6975355909999053, + 3.759505165999144, + 3.805389749999449, + 3.856281216998468, + 3.914440556000045 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 73.61074299842585, + "p95": 82.62877599918284, + "max": 84.26480300113326 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.89781081074595, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.9076701309968485, + "end_to_end_tps": 65.51218281433962, + "timing_valid": true, + "per_stream_tps": [ + 66.9 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.0957801689983171, + "decode_tps": 66.89781081074595, + "prefill_tps": 146.1680444544422, + "itl_mean_ms": 14.948172262751648, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.7579949972278, + "round_gap_p50_ms": 74.00861800124403, + "round_gap_p95_ms": 82.19074699809426, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.948172262751648, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.7579949972278, + "round_gap_p50_ms": 74.00861800124403, + "round_gap_p95_ms": 82.19074699809426, + "round_gap_mean_ms": 69.30516230912127, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 146.1680444544422 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28733.906025281, + "first": 28734.00180545, + "last": 28737.813589377, + "end": 28737.813695412, + "elapsed_s": 3.9076701309968485, + "chunk_offsets_s": [ + 0.0957801689983171, + 0.15221119499983615, + 0.2059900899985223, + 0.2578440109973599, + 0.31258677699952386, + 0.387619604000065, + 0.4703122569990228, + 0.5443208750002668, + 0.618109232000279, + 0.6926124789970345, + 0.7672037060001458, + 0.8373540859975037, + 0.913144944999658, + 0.9909029399968858, + 1.0669036569997843, + 1.1437210449985287, + 1.2217954929983534, + 1.300121512998885, + 1.3774571689973527, + 1.4369911629983108, + 1.4954207649971067, + 1.5558106190001126, + 1.6151126179975108, + 1.692543590997957, + 1.75727393499983, + 1.8068538640000043, + 1.8655995659973996, + 1.9431662059978407, + 2.021537984997849, + 2.0860012039993308, + 2.1359255190000113, + 2.2000317239981086, + 2.2781906229974993, + 2.359420988999773, + 2.443070211000304, + 2.5252609579983982, + 2.6052269359970524, + 2.6782127499973285, + 2.7546597469990957, + 2.8300282579984923, + 2.9060685529984767, + 2.985942283998156, + 3.0624960779969115, + 3.138150570997823, + 3.216251329999068, + 3.2853871730003448, + 3.343698034997942, + 3.4107780159974936, + 3.4911137219969532, + 3.555534446997626, + 3.6169538619978994, + 3.6908487859982415, + 3.754215179997118, + 3.799786491999839, + 3.85083836999911, + 3.9075640959999873 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 74.00861800124403, + "p95": 82.19074699809426, + "max": 83.64922200053115 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 123.5, + "per_stream_tps": 62.6, + "ttft_s_max": 0.19, + "request": { + "ttft_s": 0.145, + "decode_tps": 62.592, + "prefill_tps": 105.954, + "itl_mean_ms": 15.978, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.003, + "round_gap_p50_ms": 76.716, + "round_gap_p95_ms": 86.384, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 143.801, + "itl_mean_ms": 16.006, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.993, + "round_gap_p50_ms": 76.508, + "round_gap_p95_ms": 86.347, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 121.2107374756229, + "makespan_s": 4.2240482209999755, + "steady_tps": 125.4, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 123.54588791265309, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.2240482209999755, + "end_to_end_tps": 121.2107374756229, + "timing_valid": true, + "steady_tps": 125.38171416391037, + "per_stream_tps": [ + 63.2, + 61.8 + ], + "ttft_s": [ + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18850521000058507, + "decode_tps": 63.1974266485724, + "prefill_tps": 74.26850430264791, + "itl_mean_ms": 15.823429101959952, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.01408099901164, + "round_gap_p50_ms": 77.42129600228509, + "round_gap_p95_ms": 86.44613599972217, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09595700099816895, + "decode_tps": 61.77294395632654, + "prefill_tps": 145.89868226777062, + "itl_mean_ms": 16.18831701961622, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.99275900167413, + "round_gap_p50_ms": 77.64868900267174, + "round_gap_p95_ms": 86.64231299917446, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 16.005873060788087, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.99275900167413, + "round_gap_p50_ms": 77.64868900267174, + "round_gap_p95_ms": 86.44613599972217, + "round_gap_mean_ms": 72.88388625894575, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 148.45220103355933 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28737.815168408, + "first": 28738.003673618, + "last": 28742.038648039, + "end": 28742.038755213, + "elapsed_s": 4.223586805001105, + "chunk_offsets_s": [ + 0.18850521000058507, + 0.2427207289983926, + 0.29942265099816723, + 0.35180260999914026, + 0.40579874900140567, + 0.4803483680007048, + 0.5606524879985955, + 0.6384054939990165, + 0.7165186540005379, + 0.7958587059983984, + 0.8742751799982216, + 0.9516964760005067, + 1.0313098580008955, + 1.1139640889996372, + 1.1922206800009008, + 1.272813324998424, + 1.3558277049996832, + 1.4422738409994054, + 1.5220662350002385, + 1.5834458710014587, + 1.6354195790008816, + 1.6972950899980788, + 1.7636954720001086, + 1.8547135019980487, + 1.9343878949985083, + 2.0019580849984777, + 2.0567219059994386, + 2.1105189390000305, + 2.1754064630003995, + 2.252486604000296, + 2.3194406340007845, + 2.3724726760010526, + 2.43649352400098, + 2.5175874499982456, + 2.5997466379994876, + 2.683033196000906, + 2.7691421679992345, + 2.8534785009978805, + 2.93301431099826, + 3.013898204000725, + 3.09227306899993, + 3.1711170800008404, + 3.2534454349988664, + 3.332558946000063, + 3.4135730269990745, + 3.500371042999177, + 3.5746256439997524, + 3.627402707999863, + 3.691570135000802, + 3.7727078959978826, + 3.8439489839984162, + 3.900658055998065, + 3.9665768160011794, + 4.039255849998881, + 4.103326512999047, + 4.153918195999722, + 4.223479631000373 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 77.58715100135305, + "p95": 86.44613599972217, + "max": 91.01802999794018 + } + }, + { + "start": 28737.815060719, + "first": 28737.91101772, + "last": 28742.03903856, + "end": 28742.03910894, + "elapsed_s": 4.2240482209999755, + "chunk_offsets_s": [ + 0.09595700099816895, + 0.24276166799972998, + 0.29932429199834587, + 0.35172551600044244, + 0.40596009799992316, + 0.4803975460017682, + 0.5608094839990372, + 0.6384581730017089, + 0.7166843880004308, + 0.7958860729995649, + 0.8740887099993415, + 0.9513962290002382, + 1.0312754650003626, + 1.1136567380017368, + 1.1923905959993135, + 1.272765622001316, + 1.356175954999344, + 1.44253518700134, + 1.521817626999109, + 1.58375009899828, + 1.6357537160001812, + 1.6971439970002393, + 1.7638567020003393, + 1.8546171870002581, + 1.934752279001259, + 2.0021499969989236, + 2.0569296419998864, + 2.1104746960008924, + 2.1753150369986543, + 2.252442182998493, + 2.3197741270014376, + 2.3728123319997394, + 2.4366610260003654, + 2.517469399001129, + 2.599799649000488, + 2.6830239549999533, + 2.7691328299988527, + 2.853477862001455, + 2.9332750079993275, + 3.0135946160007734, + 3.092026916998293, + 3.1716636890014342, + 3.2534893509982794, + 3.3325704529997893, + 3.4140676710012485, + 3.500709984000423, + 3.575250802001392, + 3.627579127998615, + 3.6917154939983448, + 3.772708253000019, + 3.8443201080008294, + 3.900518867001665, + 3.966992380999727, + 4.039822771999752, + 4.103182531998755, + 4.153834362001362, + 4.2239778410003055 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 77.92566300122417, + "p95": 86.64231299917446, + "max": 146.80466700156103 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 123.4352439353097, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.229807176998293, + "end_to_end_tps": 121.04570694954086, + "timing_valid": true, + "steady_tps": 125.39638619346243, + "per_stream_tps": [ + 63.2, + 61.7 + ], + "ttft_s": [ + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19455341399952886, + "decode_tps": 63.19723028950046, + "prefill_tps": 71.95967273046107, + "itl_mean_ms": 15.823478266675545, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.49597900026129, + "round_gap_p50_ms": 76.42705900070723, + "round_gap_p95_ms": 86.32105799915735, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.0979792929974792, + "decode_tps": 61.7248622808267, + "prefill_tps": 142.88733437135733, + "itl_mean_ms": 16.200927196084248, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.7450059983239, + "round_gap_p50_ms": 76.80911499846843, + "round_gap_p95_ms": 86.97937800025102, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 16.012202731379897, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.54046699908213, + "round_gap_p50_ms": 76.45275899994886, + "round_gap_p95_ms": 86.34685899960459, + "round_gap_mean_ms": 72.9127088661049, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 143.80107955715508 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28742.04115125, + "first": 28742.235704664, + "last": 28746.270691622, + "end": 28746.270798421, + "elapsed_s": 4.2296471709996695, + "chunk_offsets_s": [ + 0.19455341399952886, + 0.2544943700013391, + 0.3126604839999345, + 0.3670189250005933, + 0.42010793200097396, + 0.49154001100032474, + 0.5679927700002736, + 0.6448933419997047, + 0.7227321710015531, + 0.8011910519999219, + 0.8791210150011466, + 0.9555480740018538, + 1.0359336090004945, + 1.1162210069996945, + 1.1947666910018597, + 1.281087749001017, + 1.3650793679989874, + 1.4471108719990298, + 1.5224412489988026, + 1.584870716000296, + 1.6366647300019395, + 1.6976904069997545, + 1.7636836509991554, + 1.8440143770021677, + 1.9239902080007596, + 1.9926584320019174, + 2.0456237959988357, + 2.0981936550015234, + 2.163750085001084, + 2.2481043209991185, + 2.3166307770006824, + 2.3698069429992756, + 2.4304816799995024, + 2.506315893999272, + 2.5879475080000702, + 2.671841534000123, + 2.7566974330002267, + 2.843772872001864, + 2.9242688510021253, + 3.004388953999296, + 3.082874546002131, + 3.1619265170011204, + 3.2497067129988864, + 3.330930050000461, + 3.411146421000012, + 3.4931250699992233, + 3.5685861769998155, + 3.621449324000423, + 3.685502615000587, + 3.7659725279991108, + 3.836500198998692, + 3.893701355998928, + 3.959737606000999, + 4.034212045000459, + 4.096707824999612, + 4.148107191998861, + 4.229540372001793 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.43990900032804, + "p95": 86.32105799915735, + "max": 87.78019599776599 + } + }, + { + "start": 28742.040991244, + "first": 28742.138970537, + "last": 28746.270206972, + "end": 28746.270328803, + "elapsed_s": 4.2293375589979405, + "chunk_offsets_s": [ + 0.0979792929974792, + 0.2542664859975048, + 0.3125636559998384, + 0.36710308200053987, + 0.42010833300082595, + 0.49205797799731954, + 0.5678978020005161, + 0.6447311060001084, + 0.7232758489990374, + 0.8012698239981546, + 0.879030149000755, + 0.9558392639992235, + 1.0363797309983056, + 1.1161943949991837, + 1.1948352639992663, + 1.281182122998871, + 1.364833040999656, + 1.4471291589979955, + 1.5228538579976885, + 1.584770966997894, + 1.6371358519973, + 1.6981632450006146, + 1.7638822449989675, + 1.8446272509972914, + 1.9242692820007505, + 1.9930804949981393, + 2.0458722340008535, + 2.0985055309974996, + 2.164020262000122, + 2.2480275529997016, + 2.317038778997812, + 2.370385588998033, + 2.4308842639984505, + 2.5067331149984966, + 2.587823334000859, + 2.672310858997662, + 2.757198485000117, + 2.8441778630003682, + 2.924608684999839, + 3.004091913000593, + 3.083351073997619, + 3.1619461589980347, + 3.250161665997439, + 3.330990403999749, + 3.4115213259974553, + 3.492905528000847, + 3.568490377998387, + 3.621346215997619, + 3.685937018999539, + 3.766436504000012, + 3.8368542509997496, + 3.8936426990003383, + 3.959614604998933, + 4.034059474997775, + 4.0971107090008445, + 4.148690819998592, + 4.229215727998962 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.82120949903037, + "p95": 86.97937800025102, + "max": 156.28719300002558 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 123.95233312805136, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.216274672999134, + "end_to_end_tps": 121.43421378090686, + "timing_valid": true, + "steady_tps": 125.84239011508991, + "per_stream_tps": [ + 63.4, + 62.0 + ], + "ttft_s": [ + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.1947435129986843, + "decode_tps": 63.418278155174086, + "prefill_tps": 71.88942925197506, + "itl_mean_ms": 15.768324670581006, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.01600000009057, + "round_gap_p50_ms": 75.83420900118654, + "round_gap_p95_ms": 85.06928299902938, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.10171535799963749, + "decode_tps": 61.9870040021548, + "prefill_tps": 137.63899842981328, + "itl_mean_ms": 16.132413819600604, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.10890900206869, + "round_gap_p50_ms": 76.62380599867902, + "round_gap_p95_ms": 85.44445500228903, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.950369245090805, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.01600000009057, + "round_gap_p50_ms": 76.50835599997663, + "round_gap_p95_ms": 85.06928299902938, + "round_gap_mean_ms": 72.63114566960992, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 143.38573657026436 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28746.272323943, + "first": 28746.467067456, + "last": 28750.487990247, + "end": 28750.488064686, + "elapsed_s": 4.215740742998605, + "chunk_offsets_s": [ + 0.1947435129986843, + 0.24971075499706785, + 0.3060650519983028, + 0.3604246429968043, + 0.41308439299973543, + 0.4835174129984807, + 0.5589046619970759, + 0.6379908229973807, + 0.7152530389976164, + 0.7921034079990932, + 0.8692038939989288, + 0.9450381030001154, + 1.030573268999433, + 1.1140269389979949, + 1.1937649059982505, + 1.2724346359973424, + 1.3562870129971998, + 1.4379750179978146, + 1.5135961559972202, + 1.5751590039981238, + 1.6289397459986503, + 1.6910767310000665, + 1.7584083909969195, + 1.8388880819984479, + 1.918739774999267, + 1.9913086489978014, + 2.048103203997016, + 2.1004724109989183, + 2.1659725039971818, + 2.2433398779976415, + 2.3100721089977014, + 2.3627508589997888, + 2.4231474259977404, + 2.499655781997717, + 2.5815364709997084, + 2.6652396459976444, + 2.750308928996674, + 2.833301238999411, + 2.912115408998943, + 2.9974082070002623, + 3.0799786719981057, + 3.1603865349970874, + 3.2426448129990604, + 3.323731644999498, + 3.4041788699978497, + 3.4845170919979864, + 3.5571192809984495, + 3.610488804999477, + 3.6759042119992955, + 3.756920211999386, + 3.82636824599831, + 3.8826602999979514, + 3.948459819999698, + 4.028448043998651, + 4.093447316998208, + 4.145343648997368, + 4.215666303996841 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.17128250058158, + "p95": 85.06928299902938, + "max": 85.53516599931754 + } + }, + { + "start": 28746.271790013, + "first": 28746.373505371, + "last": 28750.487270895, + "end": 28750.487351229, + "elapsed_s": 4.215561215998605, + "chunk_offsets_s": [ + 0.10171535799963749, + 0.25009948699880624, + 0.3063582359973225, + 0.3606702609977219, + 0.41396923399952357, + 0.48383889899923815, + 0.5595151699999406, + 0.6379985409985238, + 0.7156519829986792, + 0.7922757889973582, + 0.8694348749995697, + 0.945513080998353, + 1.030957536000642, + 1.1148226199984492, + 1.1935763639994548, + 1.2732872829983535, + 1.357126391998463, + 1.4384482809982728, + 1.513697422997211, + 1.5751530889974674, + 1.6290807369987306, + 1.6912793739975314, + 1.7580376250007248, + 1.8386634349990345, + 1.9188479369986453, + 1.991973734999192, + 2.0486996519975946, + 2.1007808949980245, + 2.165706744999625, + 2.243115952998778, + 2.3102593229996273, + 2.3629624749992217, + 2.4230355420004344, + 2.4998143339980743, + 2.5813909669996065, + 2.6654535719972046, + 2.7504069809983775, + 2.8333541770007287, + 2.9122279429975606, + 2.997870397997758, + 3.0802264970006945, + 3.1611207649984863, + 3.242989865000709, + 3.3237984089973907, + 3.4043751889985288, + 3.4848890579996805, + 3.5575038690003566, + 3.611285216997203, + 3.675951957997313, + 3.7570608669993817, + 3.826556747997529, + 3.883139237997966, + 3.949145447000774, + 4.029217864997918, + 4.094046267000522, + 4.145281219000026, + 4.215480881997792 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.70129899815947, + "p95": 85.44445500228903, + "max": 148.38412899916875 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 210.6, + "per_stream_tps": 54.4, + "ttft_s_max": 0.39, + "request": { + "ttft_s": 0.239, + "decode_tps": 54.372, + "prefill_tps": 60.828, + "itl_mean_ms": 18.393, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.246, + "round_gap_p50_ms": 87.988, + "round_gap_p95_ms": 97.854, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 146.195, + "itl_mean_ms": 18.436, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.256, + "round_gap_p50_ms": 88.188, + "round_gap_p95_ms": 97.062, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 207.19322927823077, + "makespan_s": 4.942246441001771, + "steady_tps": 220.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 211.58727258806982, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.91794959100298, + "end_to_end_tps": 208.21685563295142, + "timing_valid": true, + "steady_tps": 221.78124666394217, + "per_stream_tps": [ + 52.9, + 56.2, + 55.1, + 54.0 + ], + "ttft_s": [ + 0.1, + 0.38, + 0.29, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09655383100107429, + "decode_tps": 52.896818147017456, + "prefill_tps": 144.99683601207116, + "itl_mean_ms": 18.904728772544974, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.32090300065465, + "round_gap_p50_ms": 87.9604969995853, + "round_gap_p95_ms": 97.89089599871659, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3814266939989466, + "decode_tps": 56.23114165452096, + "prefill_tps": 36.70430051242996, + "itl_mean_ms": 17.78373994509856, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.05448799891747, + "round_gap_p50_ms": 87.64489700115519, + "round_gap_p95_ms": 97.06197900231928, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.286119038999459, + "decode_tps": 55.06631383952277, + "prefill_tps": 48.93068300857278, + "itl_mean_ms": 18.159922650974135, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.81109299970558, + "round_gap_p50_ms": 87.88321599786286, + "round_gap_p95_ms": 97.88475799723528, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19148429600318195, + "decode_tps": 53.961643033173694, + "prefill_tps": 73.11304525864281, + "itl_mean_ms": 18.53168183528503, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.73045200077468, + "round_gap_p50_ms": 88.41686500090873, + "round_gap_p95_ms": 97.74864499922842, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.345018300975674, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.80351800023345, + "round_gap_p50_ms": 88.1878020009026, + "round_gap_p95_ms": 97.06197900231928, + "round_gap_mean_ms": 83.53535119194281, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 146.50013709987968 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28750.489289235, + "first": 28750.585843066, + "last": 28755.406548903, + "end": 28755.406753123, + "elapsed_s": 4.9174638879994745, + "chunk_offsets_s": [ + 0.09655383100107429, + 0.44615779100058717, + 0.5073415100014245, + 0.5662256319992593, + 0.6230220680008642, + 0.7026900589989964, + 0.7933923780001351, + 0.8843115880008554, + 0.9742841070001305, + 1.062625450998894, + 1.150461863999226, + 1.2405026179985725, + 1.3328235209992272, + 1.4252445099991746, + 1.51320500699876, + 1.6030734320011106, + 1.6973295889984001, + 1.7935136939995573, + 1.8796920220011089, + 1.947151363001467, + 2.004574913000397, + 2.07104043399886, + 2.143633908999618, + 2.2358358609999414, + 2.3263905409985455, + 2.40622505400097, + 2.4674135419991217, + 2.52478289300052, + 2.5945673440000974, + 2.6809636229991156, + 2.7603897810004128, + 2.8210922250000294, + 2.8867800929983787, + 2.975364763999096, + 3.065387901999202, + 3.1593354910000926, + 3.257303448001039, + 3.355194343999756, + 3.44776212600118, + 3.540638807000505, + 3.6291451459983364, + 3.7189119050017325, + 3.815660164000292, + 3.9066148929996416, + 3.9960206569994625, + 4.086208527001872, + 4.166833667000901, + 4.22501386300064, + 4.295088113001839, + 4.388235457001429, + 4.470589183001721, + 4.533216823001567, + 4.604932709000423, + 4.688170091001666, + 4.7627402049984084, + 4.820463490999828, + 4.917259668000042 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.15092049917439, + "p95": 97.89089599871659, + "max": 349.6039599995129 + } + }, + { + "start": 28750.49002326, + "first": 28750.871449954, + "last": 28755.40630364, + "end": 28755.407047675, + "elapsed_s": 4.917024414997286, + "chunk_offsets_s": [ + 0.3814266939989466, + 0.44471812099800445, + 0.5067413100005069, + 0.565322959999321, + 0.6224805870006094, + 0.7024217869984568, + 0.7925255689988262, + 0.8834455799988064, + 0.9738878879979893, + 1.062075689998892, + 1.1501001939977868, + 1.2400995749994763, + 1.332825983998191, + 1.425036264998198, + 1.5126811619993532, + 1.6029158299970732, + 1.697438543997123, + 1.7930927380002686, + 1.8790552309983468, + 1.946920037997188, + 2.004348103997472, + 2.0706942090000666, + 2.1436110510003346, + 2.234665538999252, + 2.3252552559970354, + 2.4052745519984455, + 2.466805795997061, + 2.5245165749984153, + 2.594481756997993, + 2.679784263997135, + 2.759755065999343, + 2.819970504999219, + 2.8858906729983573, + 2.974836581997806, + 3.065106640999147, + 3.159019205999357, + 3.2567395839978417, + 3.355021586998191, + 3.4473031140005332, + 3.5400014119986736, + 3.6286151589993096, + 3.7180873029974464, + 3.8151492819997657, + 3.90591287999996, + 3.995626710999204, + 4.084802494999167, + 4.166418870998314, + 4.223812173000624, + 4.294022529000358, + 4.3879339569975855, + 4.46969555099713, + 4.5327010299988615, + 4.604677936000371, + 4.687895732997276, + 4.761588195997319, + 4.819460423997953, + 4.91628037999908 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.83470050002506, + "p95": 97.06197900231928, + "max": 98.2820030003495 + } + }, + { + "start": 28750.489489453, + "first": 28750.775608492, + "last": 28755.406388768, + "end": 28755.407147343, + "elapsed_s": 4.917657890000555, + "chunk_offsets_s": [ + 0.286119038999459, + 0.44577678499990725, + 0.5074419849988772, + 0.5661453520006035, + 0.6233585109985142, + 0.7031932919999235, + 0.7932246589989518, + 0.8840269100001024, + 0.9744805909977003, + 1.062834219999786, + 1.1507174359976489, + 1.2408192479997524, + 1.333187119998911, + 1.4258104910004477, + 1.5132737329986412, + 1.6032509559991013, + 1.6977596399992763, + 1.793192873999942, + 1.8793582289981714, + 1.9476078410007176, + 2.0045532750009443, + 2.071332297000481, + 2.1437264989981486, + 2.235537591997854, + 2.326311397999234, + 2.4059027199982665, + 2.467375157000788, + 2.5251983779999136, + 2.59523948900096, + 2.6804439189982077, + 2.759956942998542, + 2.8207467169995653, + 2.8865022499994666, + 2.9755817410004965, + 3.065500892000273, + 3.15949987500062, + 3.257384632997855, + 3.3561643680004636, + 3.447967886000697, + 3.5406561109994072, + 3.629610706000676, + 3.7187935519978055, + 3.815779462998762, + 3.906507749998127, + 3.9959941300003265, + 4.085727370998939, + 4.1668148589997145, + 4.224558312998852, + 4.2950230879978335, + 4.388220515000285, + 4.470308048999868, + 4.533358780998242, + 4.605127783997887, + 4.688269389000197, + 4.762618998000107, + 4.8201063409978815, + 4.916899314997863 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.11842249997426, + "p95": 97.88475799723528, + "max": 159.65774600044824 + } + }, + { + "start": 28750.489197752, + "first": 28750.680682048, + "last": 28755.406260916, + "end": 28755.406620323, + "elapsed_s": 4.917422571001225, + "chunk_offsets_s": [ + 0.19148429600318195, + 0.4457762170022761, + 0.5076455270027509, + 0.5658640159999777, + 0.6235451060019841, + 0.7031999320024624, + 0.7933882050019747, + 0.8845728710002732, + 0.9745318720015348, + 1.0629487370024435, + 1.15119689200219, + 1.2406781120007508, + 1.333698099002504, + 1.4254285510032787, + 1.5139025750031578, + 1.6034409860003507, + 1.6976863000018056, + 1.793027585001255, + 1.879545504001726, + 1.9475077700008114, + 2.0052798800024902, + 2.07189119600298, + 2.1442804650032485, + 2.235411527002725, + 2.326770297000621, + 2.4061352130011073, + 2.46790580400193, + 2.5251637700021092, + 2.595027208000829, + 2.680695053000818, + 2.7601776650008105, + 2.820950063000055, + 2.8871933050031657, + 2.9757724030023383, + 3.0658630280013313, + 3.159755413002131, + 3.2575040580013592, + 3.3559613730030833, + 3.4479740360002324, + 3.5410139190025802, + 3.629573441001412, + 3.7187948180035164, + 3.815595508000115, + 3.9064580270023725, + 3.996227972002089, + 4.085671049000666, + 4.167209259001538, + 4.224697061999905, + 4.295256225002959, + 4.388887524000893, + 4.47030900900063, + 4.533373240999936, + 4.605351252001128, + 4.688355349000631, + 4.762631199002499, + 4.820165766002901, + 4.917063164000865 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.44544450039393, + "p95": 97.74864499922842, + "max": 254.29192099909415 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 210.41816545610345, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.945227033997071, + "end_to_end_tps": 207.06834953386016, + "timing_valid": true, + "steady_tps": 220.6220057130065, + "per_stream_tps": [ + 55.9, + 53.7, + 54.8, + 52.6 + ], + "ttft_s": [ + 0.39, + 0.19, + 0.29, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.3854569499999343, + "decode_tps": 55.9347836642273, + "prefill_tps": 36.32052814199455, + "itl_mean_ms": 17.87796312939962, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.12393200141378, + "round_gap_p50_ms": 87.96411499861279, + "round_gap_p95_ms": 96.26262500023586, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19250383600228815, + "decode_tps": 53.67502992661659, + "prefill_tps": 72.72582349908909, + "itl_mean_ms": 18.630637027444227, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.57671800151002, + "round_gap_p50_ms": 88.4391460022016, + "round_gap_p95_ms": 97.82256799735478, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2890249810006935, + "decode_tps": 54.782923484244634, + "prefill_tps": 48.4387195581769, + "itl_mean_ms": 18.253863364696052, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.53571899898816, + "round_gap_p50_ms": 88.5489289976249, + "round_gap_p95_ms": 97.67432200169424, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.0969714919992839, + "decode_tps": 52.606711586520206, + "prefill_tps": 144.37232748881894, + "itl_mean_ms": 19.008981360778254, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.17141800036188, + "round_gap_p50_ms": 88.81883999856655, + "round_gap_p95_ms": 97.92142000151216, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.442861220579537, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.34421500150347, + "round_gap_p50_ms": 88.5489289976249, + "round_gap_p95_ms": 96.50115700060269, + "round_gap_mean_ms": 83.98088591513897, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 145.2353276282134 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28755.408596755, + "first": 28755.794053705, + "last": 28760.352934303, + "end": 28760.352996399, + "elapsed_s": 4.944399643998622, + "chunk_offsets_s": [ + 0.3854569499999343, + 0.44811183699857793, + 0.5096445459967072, + 0.5678709479980171, + 0.62513241799752, + 0.7025520989991492, + 0.7906628229975468, + 0.8858605239984172, + 0.9774193469966121, + 1.0647878939998918, + 1.1516275369976938, + 1.2407295489974786, + 1.3331891729976633, + 1.4243733579969557, + 1.5149371099978453, + 1.607061041999259, + 1.7026209509967885, + 1.7932566279996536, + 1.883364437999262, + 1.954071746997215, + 2.0106511139965733, + 2.0787869039995712, + 2.15021816999797, + 2.2456937589995505, + 2.3370083879999584, + 2.41671509599837, + 2.4782543949986575, + 2.535448106998956, + 2.605142717999115, + 2.693106832997728, + 2.7678475579996302, + 2.8259532249976473, + 2.8965083599978243, + 2.9855903909992776, + 3.0774505769986717, + 3.1695042829996964, + 3.2681683139999222, + 3.3657785399991553, + 3.4592047139994975, + 3.554627781999443, + 3.6452286029998504, + 3.7345648709997477, + 3.8269230499972764, + 3.9231856749975123, + 4.012766918996931, + 4.102962700999342, + 4.187701819999347, + 4.2472163809979975, + 4.316871254999569, + 4.411571792999894, + 4.495106360998761, + 4.560049004998291, + 4.632031432996882, + 4.7175098640000215, + 4.787906548997853, + 4.849059565996868, + 4.944337547996838 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.03741949850519, + "p95": 96.26262500023586, + "max": 98.6640310002258 + } + }, + { + "start": 28755.409376196, + "first": 28755.601880032, + "last": 28760.352692474, + "end": 28760.353439952, + "elapsed_s": 4.944063756000105, + "chunk_offsets_s": [ + 0.19250383600228815, + 0.4471460710010433, + 0.5087083710022853, + 0.5669874550003442, + 0.6238512960007938, + 0.7015145959994697, + 0.7899537420016713, + 0.8853113949990075, + 0.9768093700004101, + 1.0635143300023628, + 1.1505597050017968, + 1.239831132999825, + 1.3323217409997596, + 1.4232508019995294, + 1.5135909999989963, + 1.6059352150004997, + 1.70165627499955, + 1.7921034800019697, + 1.8828239270005724, + 1.9530097650022071, + 2.0101569879989256, + 2.078227345002233, + 2.150326932001917, + 2.2448434510006336, + 2.3358424620018923, + 2.4156709050002974, + 2.4780640280005173, + 2.5348069890023908, + 2.6047896910022246, + 2.691803728001105, + 2.767128973999206, + 2.825409868000861, + 2.89542425000036, + 2.9845868410011462, + 3.076605516998825, + 3.1695615429998725, + 3.267617182002141, + 3.3654397499994957, + 3.4580164680010057, + 3.5534416520022205, + 3.644236808999267, + 3.73341291700126, + 3.8260317660024157, + 3.9220269399993413, + 4.01156073400125, + 4.102300665999792, + 4.186488164999901, + 4.245978841001488, + 4.315691873001924, + 4.411190110000462, + 4.494264470999042, + 4.55907260100139, + 4.631575473002158, + 4.717002641998988, + 4.786360042999149, + 4.848105249002401, + 4.943316278000566 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.80086850149382, + "p95": 97.82256799735478, + "max": 254.64223499875516 + } + }, + { + "start": 28755.409081292, + "first": 28755.698106273, + "last": 28760.352841431, + "end": 28760.353699621, + "elapsed_s": 4.944618328998331, + "chunk_offsets_s": [ + 0.2890249810006935, + 0.44753146299990476, + 0.5088297449983656, + 0.567024184998445, + 0.624335455999244, + 0.7014562580006896, + 0.7900051869983145, + 0.8852772040008858, + 0.9774434409991954, + 1.0638914269984525, + 1.1510493409987248, + 1.2403931619992363, + 1.332550816998264, + 1.4237105940010224, + 1.5143652869992366, + 1.6062863639999705, + 1.7024304500009748, + 1.7922207820010954, + 1.8833578250014398, + 1.9534815340011846, + 2.0104069500011974, + 2.078105668999342, + 2.150401890001376, + 2.244843355998455, + 2.336352053000155, + 2.4161722030003148, + 2.478249802999926, + 2.535354682000616, + 2.6054540330005693, + 2.6924483670009067, + 2.7673085770002217, + 2.825589199001115, + 2.8958412690008117, + 2.9849507459985034, + 3.0767878779988678, + 3.169323596997856, + 3.268018007998762, + 3.3656923300004564, + 3.4582422809980926, + 3.5536793430001126, + 3.6444679439991887, + 3.7337961320008617, + 3.826635125999019, + 3.9225617709998915, + 4.012071727000148, + 4.102684025998315, + 4.186682061997999, + 4.246206128998892, + 4.316161202998046, + 4.4111955469998065, + 4.494764898001449, + 4.559816539000167, + 4.631956707999052, + 4.717344398999558, + 4.7864990169982775, + 4.848662050000712, + 4.9437601389981864 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.82920299765829, + "p95": 97.67432200169424, + "max": 158.50648199921125 + } + }, + { + "start": 28755.408472587, + "first": 28755.505444079, + "last": 28760.352734326, + "end": 28760.353309284, + "elapsed_s": 4.944836696999118, + "chunk_offsets_s": [ + 0.0969714919992839, + 0.4479523699992569, + 0.5091321569998399, + 0.5673321539979952, + 0.6245807339983003, + 0.7017840659973444, + 0.7910756539968133, + 0.8861760609979683, + 0.9783349319986883, + 1.0643489389985916, + 1.1511010889989848, + 1.2407086089988297, + 1.3325965439980791, + 1.4238825649990758, + 1.5147666159973596, + 1.6065087119968666, + 1.7020162549997622, + 1.7935664469987387, + 1.884208730996761, + 1.9540316049969988, + 2.010609848999593, + 2.078649453997059, + 2.1505104649986606, + 2.245564871998795, + 2.336515060997044, + 2.4163346939967596, + 2.4785905680000724, + 2.535630726997624, + 2.6057990949993837, + 2.6924892559982254, + 2.767843487999926, + 2.8262542719967314, + 2.896504112999537, + 2.985606246998941, + 3.077632218999497, + 3.170177011998021, + 3.268473576998076, + 3.3663949969995883, + 3.45856641499995, + 3.553957101998094, + 3.6447263219997694, + 3.7343623409979045, + 3.826856873998622, + 3.923358030999225, + 4.012176870997791, + 4.103259323997918, + 4.18759672199667, + 4.246506169998611, + 4.316694372999336, + 4.411557774998073, + 4.494967381997412, + 4.559675297998183, + 4.632598416999826, + 4.717811487997096, + 4.78733669599751, + 4.8490889709974, + 4.944261738997739 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.96048699898529, + "p95": 97.92142000151216, + "max": 350.980877999973 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 210.5715582947734, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.942246441001771, + "end_to_end_tps": 207.19322927823077, + "timing_valid": true, + "steady_tps": 220.6376045127046, + "per_stream_tps": [ + 53.7, + 55.9, + 54.8, + 52.6 + ], + "ttft_s": [ + 0.19, + 0.38, + 0.29, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.1920340889992076, + "decode_tps": 53.69469048645309, + "prefill_tps": 72.90372283880166, + "itl_mean_ms": 18.62381533332975, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.5132640004449, + "round_gap_p50_ms": 88.01165800105082, + "round_gap_p95_ms": 98.67308599859825, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38292683600229793, + "decode_tps": 55.93684886649131, + "prefill_tps": 36.560508911200955, + "itl_mean_ms": 17.877303070589036, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.74273700045887, + "round_gap_p50_ms": 87.32111299832468, + "round_gap_p95_ms": 97.55846700136317, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28771074199903524, + "decode_tps": 54.798490684608105, + "prefill_tps": 48.65998364443044, + "itl_mean_ms": 18.248677792158276, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.57334400172113, + "round_gap_p50_ms": 87.52862400069716, + "round_gap_p95_ms": 98.70197800046299, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09780330899957335, + "decode_tps": 52.646488672005184, + "prefill_tps": 143.14444105424974, + "itl_mean_ms": 18.99461911372925, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.61508200142998, + "round_gap_p50_ms": 88.11667100235354, + "round_gap_p95_ms": 98.59087600125349, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.43610382745158, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.25605100073153, + "round_gap_p50_ms": 88.01165800105082, + "round_gap_p95_ms": 97.55846700136317, + "round_gap_mean_ms": 83.95011564285986, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 146.1949946224801 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28760.355233013, + "first": 28760.547267102, + "last": 28765.296340012, + "end": 28765.297068184, + "elapsed_s": 4.941835171001003, + "chunk_offsets_s": [ + 0.1920340889992076, + 0.44495499699769425, + 0.5082431109985919, + 0.5677076549982303, + 0.6258262949995697, + 0.7040433490001305, + 0.7923736320008175, + 0.8826763559991377, + 0.979900597998494, + 1.0674578840007598, + 1.1544871879996208, + 1.24511525699927, + 1.3376414459999069, + 1.4267204759999004, + 1.5151072699991346, + 1.605053463998047, + 1.6985497790010413, + 1.7886062670004321, + 1.8732569509993482, + 1.9464607250010886, + 2.0058915970003, + 2.074217012999725, + 2.148004836999462, + 2.2398411850008415, + 2.329626233000454, + 2.410533293001208, + 2.471828974001255, + 2.530484348000755, + 2.6025279109999246, + 2.6905395690009755, + 2.7646863179979846, + 2.822528714998043, + 2.891419532999862, + 2.983932797000307, + 3.0745492489986646, + 3.1675775650001015, + 3.2662506509986997, + 3.3626570269989315, + 3.4548813840010553, + 3.548180208999838, + 3.6382010159977654, + 3.728645790997689, + 3.8193468079989543, + 3.9125902369996766, + 4.005445063998195, + 4.096427835000213, + 4.178405673999805, + 4.236242815000878, + 4.306436906001181, + 4.401144335999561, + 4.483207430999755, + 4.548800701999426, + 4.621577111000079, + 4.705605661998561, + 4.775828908001131, + 4.8324347359994135, + 4.941106998998293 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.17097050086886, + "p95": 98.67308599859825, + "max": 252.92090799848665 + } + }, + { + "start": 28760.354944957, + "first": 28760.737871793, + "last": 28765.296584076, + "end": 28765.296781434, + "elapsed_s": 4.941836477002653, + "chunk_offsets_s": [ + 0.38292683600229793, + 0.4458805520007445, + 0.5086888300029386, + 0.5677819510019617, + 0.6258086880006886, + 0.704036144001293, + 0.7926002060012252, + 0.8828944830020191, + 0.9804529500033823, + 1.0679765940003563, + 1.1546948170034739, + 1.2454535430006217, + 1.3379669910027587, + 1.4268606770019687, + 1.5153533530028653, + 1.6051042810031504, + 1.6988891680011875, + 1.788809030000266, + 1.874002897002356, + 1.946909702000994, + 2.0060960430018895, + 2.074420866003493, + 2.1487610890035285, + 2.24023027200019, + 2.330090099003428, + 2.411064117000933, + 2.4720206540005165, + 2.5309719820033934, + 2.6035620550028398, + 2.6908831680011644, + 2.7649263300008897, + 2.822742990003462, + 2.8916679740032123, + 2.9843527890006953, + 3.074887287002639, + 3.1679176710022148, + 3.2669022560003214, + 3.3631351590011036, + 3.4548778960015625, + 3.5487625650021073, + 3.638410425002803, + 3.728506482002558, + 3.8195897720033827, + 3.913231963000726, + 4.005410762001702, + 4.096553229002893, + 4.178522159003478, + 4.236450702002912, + 4.307129972999974, + 4.401587875003315, + 4.483562511002674, + 4.548750573001598, + 4.621919967001304, + 4.705706808003015, + 4.776079021001351, + 4.832665397003439, + 4.941639119002502 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.42237849764933, + "p95": 97.55846700136317, + "max": 108.97372199906386 + } + }, + { + "start": 28760.355064693, + "first": 28760.642775435, + "last": 28765.296188272, + "end": 28765.296961967, + "elapsed_s": 4.94189727399862, + "chunk_offsets_s": [ + 0.28771074199903524, + 0.4449112080001214, + 0.5083365809987299, + 0.5673279579968948, + 0.6254184379977232, + 0.703562470000179, + 0.7921319649976795, + 0.8829296059993794, + 0.980199959998572, + 1.0677208629967936, + 1.1550087889991119, + 1.2450753779994557, + 1.338179266997031, + 1.4270785949993297, + 1.5154547339989222, + 1.6050747739973303, + 1.698402490997978, + 1.788968875000137, + 1.87374655199892, + 1.9463970309989236, + 2.005898610997974, + 2.0746484379997128, + 2.147957328998018, + 2.240247048997844, + 2.3295781969973177, + 2.4103760839971073, + 2.47153493299993, + 2.530395237998164, + 2.6030096689974016, + 2.6905382929980988, + 2.765033607996884, + 2.822445149999112, + 2.891448688998935, + 2.9843427779997, + 3.0748814589969697, + 3.167454802998691, + 3.266156780999154, + 3.362516876997688, + 3.4546333799989952, + 3.5480858769988117, + 3.6378128319993266, + 3.7287130299991986, + 3.819356023999717, + 3.912868546998652, + 4.005463830999361, + 4.0965796659984335, + 4.178213449999021, + 4.236117747997923, + 4.306501469000068, + 4.40112222499738, + 4.483101433997945, + 4.548184815997956, + 4.621503705999203, + 4.705502789998718, + 4.775603682999645, + 4.8323450190000585, + 4.941123578999395 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.95238150014484, + "p95": 98.70197800046299, + "max": 157.20046600108617 + } + }, + { + "start": 28760.354821743, + "first": 28760.452625052, + "last": 28765.296252926, + "end": 28765.296399169, + "elapsed_s": 4.941577426001459, + "chunk_offsets_s": [ + 0.09780330899957335, + 0.4458018660006928, + 0.5083519229992817, + 0.5675124879999203, + 0.625595098001213, + 0.7037510859991016, + 0.7925820080017729, + 0.8831455979998282, + 0.9804932369988819, + 1.068044015999476, + 1.1551405069985776, + 1.2454292549991806, + 1.3382074309993186, + 1.427422470998863, + 1.5157572220014117, + 1.6053829670017876, + 1.698594638000941, + 1.7890727480007627, + 1.8742158269997162, + 1.946431141001085, + 2.0060502580017783, + 2.0742526079993695, + 2.1482641129987314, + 2.240430492998712, + 2.3297514320001937, + 2.4105520920020354, + 2.4717304650002916, + 2.530714500000613, + 2.6027330799988704, + 2.690849751001224, + 2.7653285919986956, + 2.8226317709995783, + 2.8916111440012173, + 2.9845430530003796, + 3.0750777590001235, + 3.1677660519999336, + 3.266356928001187, + 3.3627225359996373, + 3.4553376180010673, + 3.5489620699991065, + 3.638110249001329, + 3.7293916779999563, + 3.8194841770018684, + 3.913398666998546, + 4.005654717999278, + 4.096755683000083, + 4.178512276001129, + 4.236307723000209, + 4.306680033998418, + 4.401302936999855, + 4.4833884479994595, + 4.548631788999046, + 4.621836802001781, + 4.705500954001764, + 4.775893403999362, + 4.832998467001744, + 4.941431183000532 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 88.22571100245113, + "p95": 98.59087600125349, + "max": 347.9985570011195 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 241.8, + "per_stream_tps": 50.2, + "ttft_s_max": 0.49, + "request": { + "ttft_s": 0.289, + "decode_tps": 50.188, + "prefill_tps": 48.464, + "itl_mean_ms": 19.925, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.784, + "round_gap_p50_ms": 93.695, + "round_gap_p95_ms": 106.502, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 144.316, + "itl_mean_ms": 19.931, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.737, + "round_gap_p50_ms": 94.01, + "round_gap_p95_ms": 105.245, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 238.27954470797275, + "makespan_s": 5.3718417230011255, + "steady_tps": 256.9, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 242.02112182801477, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.367129595000733, + "end_to_end_tps": 238.48874474584494, + "timing_valid": true, + "steady_tps": 257.2546617482066, + "per_stream_tps": [ + 52.2, + 48.4, + 50.2, + 51.3, + 49.3 + ], + "ttft_s": [ + 0.48, + 0.1, + 0.29, + 0.39, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.48494316500000423, + "decode_tps": 52.23732996364795, + "prefill_tps": 28.86936245405145, + "itl_mean_ms": 19.143398039216432, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.4685820005543, + "round_gap_p50_ms": 93.27274199677049, + "round_gap_p95_ms": 105.03043999779038, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09815593700113823, + "decode_tps": 48.40586685174314, + "prefill_tps": 142.63019056949815, + "itl_mean_ms": 20.658652866661534, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.67967400007183, + "round_gap_p50_ms": 93.26980900004855, + "round_gap_p95_ms": 105.80707099870779, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2853524220008694, + "decode_tps": 50.18762932572163, + "prefill_tps": 49.06213832647036, + "itl_mean_ms": 19.925228854902112, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.1344260002952, + "round_gap_p50_ms": 93.1029329985904, + "round_gap_p95_ms": 105.62729799858062, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38996777899956214, + "decode_tps": 51.251024288129415, + "prefill_tps": 35.900401915040575, + "itl_mean_ms": 19.511805156870132, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.60154600048554, + "round_gap_p50_ms": 93.28227500009234, + "round_gap_p95_ms": 105.4847070008691, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19192847999875084, + "decode_tps": 49.27883751958676, + "prefill_tps": 72.9438382468882, + "itl_mean_ms": 20.2926864823573, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.00308000235236, + "round_gap_p50_ms": 93.10903799996595, + "round_gap_p95_ms": 106.0763629975554, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.9063542800015, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.73683200223604, + "round_gap_p50_ms": 93.26980900004855, + "round_gap_p95_ms": 105.28577399963979, + "round_gap_mean_ms": 90.64500609643541, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 144.31611308197623 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28765.29827928, + "first": 28765.783222445, + "last": 28770.664788945, + "end": 28770.665047892, + "elapsed_s": 5.36676861199885, + "chunk_offsets_s": [ + 0.48494316500000423, + 0.5504430280016095, + 0.6170851589995436, + 0.679058398000052, + 0.73907013400094, + 0.8222022209993156, + 0.9116544810021878, + 1.010874526000407, + 1.111148353000317, + 1.2064084270023159, + 1.2996811689990864, + 1.3973430699988967, + 1.4994945600010396, + 1.5977450509999471, + 1.6946218080011022, + 1.7909202600021672, + 1.8937709550009458, + 1.996749981000903, + 2.0909717319991614, + 2.163598393999564, + 2.223258514000918, + 2.297239764000551, + 2.375171362000401, + 2.472618992000207, + 2.56873989900123, + 2.6577207059999637, + 2.721028586001921, + 2.781249685001967, + 2.8574487949990726, + 2.950968658002239, + 3.0393072510014463, + 3.1006262820010306, + 3.1694081900022866, + 3.260438019002322, + 3.3588467449990276, + 3.459614169001725, + 3.567538606999733, + 3.672824380999373, + 3.7733427040002425, + 3.875865974001499, + 3.9761950870015426, + 4.075663669002097, + 4.175622294998902, + 4.271644619999279, + 4.369940317999863, + 4.469400074001896, + 4.559158563999517, + 4.620188379001775, + 4.697142168999562, + 4.795604745999299, + 4.884369989998959, + 4.956012302001909, + 5.038973676000751, + 5.127459709001414, + 5.2019635880024, + 5.261479225002404, + 5.366509665000194 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.39630249996844, + "p95": 105.03043999779038, + "max": 107.92443799800822 + } + }, + { + "start": 28765.298497771, + "first": 28765.396653708, + "last": 28770.664610189, + "end": 28770.664822008, + "elapsed_s": 5.36632423699848, + "chunk_offsets_s": [ + 0.09815593700113823, + 0.5501430320000509, + 0.6173520449992793, + 0.6789710170014587, + 0.7382978829991771, + 0.8216300880012568, + 0.9113902460012469, + 1.0107166950001556, + 1.1113956279987178, + 1.2060439579981903, + 1.2993137669982389, + 1.3973307199994451, + 1.4987762999990082, + 1.5973609950015089, + 1.69408295500034, + 1.7900411609989533, + 1.8936948939990543, + 1.9964108649983245, + 2.0903319440003543, + 2.1635841589995835, + 2.222502427001018, + 2.296633732999908, + 2.3747908160003135, + 2.4717482720006956, + 2.56846786800088, + 2.656483743998251, + 2.7205773499990755, + 2.7808262440012186, + 2.8570481349997863, + 2.9502403939986834, + 3.0397095219996118, + 3.1000286980015517, + 3.16863261799881, + 3.2603021299983084, + 3.358350041999074, + 3.459932605001086, + 3.5667206590005662, + 3.672527729999274, + 3.77277173100083, + 3.875098679000075, + 3.976138074998744, + 4.0753620360010245, + 4.175041710001096, + 4.271634512999299, + 4.3693269889990916, + 4.468467231999966, + 4.558338767998066, + 4.619918803000473, + 4.6967590669992205, + 4.795192969999334, + 4.884223936998751, + 4.955531420000625, + 5.0385015239990025, + 5.1275245459983125, + 5.2018723680012044, + 5.260850278998987, + 5.36611241799983 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.59544400103914, + "p95": 105.80707099870779, + "max": 451.98709499891265 + } + }, + { + "start": 28765.298418377, + "first": 28765.583770799, + "last": 28770.664704157, + "end": 28770.665305717, + "elapsed_s": 5.366887340001995, + "chunk_offsets_s": [ + 0.2853524220008694, + 0.5510579850015347, + 0.6173816420014191, + 0.6788851100027387, + 0.7386337450006977, + 0.8222399610021967, + 0.9104951690023881, + 1.0102286470028048, + 1.1109294340021734, + 1.2066092410022975, + 1.299692104003043, + 1.397052963002352, + 1.4992614500006312, + 1.598025969000446, + 1.6942790640023304, + 1.7905056119998335, + 1.8938440600031754, + 1.9969024560014077, + 2.090005388999998, + 2.1632970820028277, + 2.2227213670012134, + 2.296826238001813, + 2.375137638002343, + 2.4720001200003026, + 2.5679526130006707, + 2.657209133001743, + 2.720478434999677, + 2.781385564001539, + 2.8575440130007337, + 2.950730922002549, + 3.039410608002072, + 3.1002475510031218, + 3.1690088880022813, + 3.2601572210005543, + 3.358770689002995, + 3.459425649001787, + 3.5671501480028383, + 3.672777446001419, + 3.773022328001389, + 3.875646650001727, + 3.976262324002164, + 4.076396750002459, + 4.174740860002203, + 4.271175740999752, + 4.369551858002524, + 4.4686396200013405, + 4.5585151890009, + 4.619837366000866, + 4.697131592001824, + 4.7954988880010205, + 4.884172280002531, + 4.95579927700237, + 5.038744953002606, + 5.127921131999756, + 5.201768825001636, + 5.260987085002853, + 5.366285780000908 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.14492100020288, + "p95": 105.62729799858062, + "max": 265.7055630006653 + } + }, + { + "start": 28765.298697837, + "first": 28765.688665616, + "last": 28770.664175931, + "end": 28770.664304627, + "elapsed_s": 5.3656067900010385, + "chunk_offsets_s": [ + 0.38996777899956214, + 0.5506688999994367, + 0.6169815979992563, + 0.6785461600011331, + 0.7385869600002479, + 0.8215744459994312, + 0.910913989002438, + 1.010074721001729, + 1.1103717320002033, + 1.2059311490011169, + 1.2992134240012092, + 1.3968666239998129, + 1.4984415110011469, + 1.5975293240007886, + 1.694067742999323, + 1.7903837539997767, + 1.8931824440005585, + 1.996425661000103, + 2.0904242510005133, + 2.1634787790026166, + 2.2226466270003584, + 2.2972562510003627, + 2.374815736999153, + 2.4719317570015846, + 2.5684100319995196, + 2.657051541002147, + 2.7209472930007905, + 2.7809066160007205, + 2.857412438999745, + 2.9505121799993503, + 3.03922664700076, + 3.099593050999829, + 3.168558569999732, + 3.2597520320014155, + 3.3582271949999267, + 3.4594470219999494, + 3.5674155980013893, + 3.6729003050022584, + 3.773005879000266, + 3.8750663080027152, + 3.9762125550005294, + 4.075814101001015, + 4.174143649001053, + 4.271336695001082, + 4.369724475000112, + 4.468133882000984, + 4.558572496000124, + 4.619370388001698, + 4.696943696999369, + 4.795509292001952, + 4.884218630002579, + 4.955418097000802, + 5.0385136299992155, + 5.126896901001601, + 5.201090444999863, + 5.260501845001272, + 5.365478094001446 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.64043250025134, + "p95": 105.4847070008691, + "max": 160.70112099987455 + } + }, + { + "start": 28765.298176122, + "first": 28765.490104602, + "last": 28770.664739655, + "end": 28770.664938018, + "elapsed_s": 5.366761895998934, + "chunk_offsets_s": [ + 0.19192847999875084, + 0.5514759229990887, + 0.617733537001186, + 0.6795301320016733, + 0.7392567679999047, + 0.8224092189993826, + 0.9105192329989222, + 1.010360555999796, + 1.1114288290009426, + 1.20655803399859, + 1.2998214020008163, + 1.3980241649987875, + 1.4996753900013573, + 1.5979323690007732, + 1.694649152999773, + 1.7904586820004624, + 1.8942259249997733, + 1.9969856669995352, + 2.090094704999501, + 2.164147731000412, + 2.2231057950011746, + 2.297621885998524, + 2.3752056769990304, + 2.472928953000519, + 2.568297558998893, + 2.6570435220019135, + 2.7213355569983833, + 2.7815012249993742, + 2.857990434000385, + 2.950604673998896, + 3.039883461999125, + 3.100781203000224, + 3.169583953000256, + 3.2608443809986056, + 3.358444215999043, + 3.4594549749999715, + 3.567288931000803, + 3.6733652939983585, + 3.773368374000711, + 3.875543951999134, + 3.976160827998683, + 4.076698421999026, + 4.175108838000597, + 4.271253569000692, + 4.36935555799937, + 4.469092390001606, + 4.559217205001914, + 4.620358240001224, + 4.697328320999077, + 4.795568170000479, + 4.884914023001329, + 4.956081484000606, + 5.039128014999733, + 5.128111598001851, + 5.202140770001279, + 5.261433357998612, + 5.366563532999862 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.18620300109615, + "p95": 106.0763629975554, + "max": 359.5474430003378 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 241.78726075581807, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.3718417230011255, + "end_to_end_tps": 238.27954470797275, + "timing_valid": true, + "steady_tps": 256.65839974468895, + "per_stream_tps": [ + 52.1, + 50.2, + 49.2, + 48.4, + 51.1 + ], + "ttft_s": [ + 0.48, + 0.29, + 0.19, + 0.1, + 0.38 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.47745683700122754, + "decode_tps": 52.1220135416297, + "prefill_tps": 29.32202225426297, + "itl_mean_ms": 19.185751509797353, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.56989100101055, + "round_gap_p50_ms": 93.5178250001627, + "round_gap_p95_ms": 104.53651100033312, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28887466300147935, + "decode_tps": 50.18438156053308, + "prefill_tps": 48.46392499271667, + "itl_mean_ms": 19.926518349016344, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.78370600219932, + "round_gap_p50_ms": 93.69524599969736, + "round_gap_p95_ms": 108.42767999929492, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.1923712199968577, + "decode_tps": 49.247387518778176, + "prefill_tps": 72.77595889982236, + "itl_mean_ms": 20.305645647064974, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.53793400200084, + "round_gap_p50_ms": 94.49430800304981, + "round_gap_p95_ms": 107.96252799991635, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09718879499996547, + "decode_tps": 48.359635291876735, + "prefill_tps": 144.04952752017323, + "itl_mean_ms": 20.67840243137599, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.7715249977773, + "round_gap_p50_ms": 93.74154299803195, + "round_gap_p95_ms": 108.68789999949513, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38320944900260656, + "decode_tps": 51.133956279807194, + "prefill_tps": 36.533545914481806, + "itl_mean_ms": 19.556476219597744, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.86969100282295, + "round_gap_p50_ms": 93.85871599806705, + "round_gap_p95_ms": 107.7500860010332, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.93055883137048, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.58416200242937, + "round_gap_p50_ms": 94.01030099979835, + "round_gap_p95_ms": 104.90073400069377, + "round_gap_mean_ms": 90.75522324999058, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 146.49926496953157 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28770.667187051, + "first": 28771.144643888, + "last": 28776.037010523, + "end": 28776.037859278, + "elapsed_s": 5.370672226999886, + "chunk_offsets_s": [ + 0.47745683700122754, + 0.5436969350012077, + 0.6148690510017332, + 0.6794961859995965, + 0.7395246449996193, + 0.8246771459998854, + 0.9143773389987473, + 1.0108912190007686, + 1.1107926320000843, + 1.2063101419989835, + 1.2998333319992526, + 1.3977600149992213, + 1.499455653000041, + 1.6006782250005926, + 1.700990321001882, + 1.798857483001484, + 1.9005948459998763, + 1.9991831379993528, + 2.0912964540002577, + 2.1653411830011464, + 2.2274174890008, + 2.301358265998715, + 2.380669399000908, + 2.4810261839993473, + 2.58063748800123, + 2.6741553130013926, + 2.7380665490018146, + 2.7984433980018366, + 2.875617183999566, + 2.969638473001396, + 3.052644227998826, + 3.114418533001299, + 3.184107249999215, + 3.276510389001487, + 3.3742378069982806, + 3.473807697999291, + 3.581736272000853, + 3.6930926780005393, + 3.794354262001434, + 3.8956258079997497, + 3.991638021001563, + 4.0882411940001475, + 4.187455104998662, + 4.285128564999468, + 4.382068126000377, + 4.479286990001128, + 4.56673368599877, + 4.632841973001632, + 4.710515637998469, + 4.807316709000588, + 4.896760643001471, + 4.967951769998763, + 5.044684945998597, + 5.130794929998956, + 5.205609699001798, + 5.2652869609992194, + 5.369823471999553 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.5205075002159, + "p95": 104.53651100033312, + "max": 111.3564059996861 + } + }, + { + "start": 28770.666914921, + "first": 28770.955789584, + "last": 28776.037051763, + "end": 28776.037185252, + "elapsed_s": 5.370270331000938, + "chunk_offsets_s": [ + 0.28887466300147935, + 0.5437364470017201, + 0.61456236000231, + 0.679305548001139, + 0.7395678409993707, + 0.8243465240011574, + 0.9144192220010154, + 1.0113495770019654, + 1.1108786129989312, + 1.2063627339994127, + 1.2998462479990849, + 1.3979378969997924, + 1.4998600079998141, + 1.601084133999393, + 1.7008678400015924, + 1.7986274270006106, + 1.9009595419993275, + 1.9987157010000374, + 2.091471677998925, + 2.165509138001653, + 2.226805942002102, + 2.3011506530019687, + 2.3803217810018396, + 2.480475214000762, + 2.5802860420008074, + 2.6739812880005047, + 2.7385824260018126, + 2.7985119830009353, + 2.8754392990013002, + 2.970246832999692, + 3.053509404999204, + 3.11489606500254, + 3.18353558400122, + 3.27619399300238, + 3.374078773002111, + 3.473649513001874, + 3.582077193001169, + 3.6932454529996903, + 3.79433132000122, + 3.8962567710004805, + 3.991555802000221, + 4.088258550000319, + 4.188394172000699, + 4.285458705002384, + 4.382120678001229, + 4.479134473000158, + 4.566899838002428, + 4.633380593000766, + 4.7111910179992265, + 4.807935824999731, + 4.897143491001771, + 4.968169798001327, + 5.045942120999825, + 5.131232163999812, + 5.205947989001288, + 5.265719261999038, + 5.3701368420006474 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 94.25138999904448, + "p95": 108.42767999929492, + "max": 254.86178400024073 + } + }, + { + "start": 28770.667090088, + "first": 28770.859461308, + "last": 28776.037400948, + "end": 28776.038021163, + "elapsed_s": 5.370931074998225, + "chunk_offsets_s": [ + 0.1923712199968577, + 0.5432024539986742, + 0.6147652879990346, + 0.6787194789976638, + 0.7391367819982406, + 0.8238765079986479, + 0.9141824159996759, + 1.0114130989968544, + 1.1103942609988735, + 1.2061027379968436, + 1.3005970459998935, + 1.39764530499815, + 1.499290181996912, + 1.6007132629965781, + 1.700765681998746, + 1.7987382389983395, + 1.9012024199982989, + 1.9987445829974604, + 2.091824163999263, + 2.165380708996963, + 2.2268399739987217, + 2.300934322000103, + 2.3803568059993268, + 2.4806794749965775, + 2.5797949230000086, + 2.674030503996619, + 2.737743624998984, + 2.7984785989974625, + 2.8749647739969078, + 2.969901228996605, + 3.0533713179975166, + 3.1144007179973414, + 3.1843771429994376, + 3.2760717929995735, + 3.3739752959991165, + 3.473513698998431, + 3.5814762269983476, + 3.692942554996989, + 3.7943719909999345, + 3.896199869999691, + 3.9914696870000625, + 4.088211626996781, + 4.187749560998782, + 4.284785009996995, + 4.382365208999545, + 4.479472648999945, + 4.566557336998812, + 4.633049446998484, + 4.710731070998008, + 4.807549551998818, + 4.897008347998053, + 4.9681040679970465, + 5.045381801999611, + 5.13096093599961, + 5.205817456997465, + 5.2654919159977, + 5.370310859998426 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 94.71538150137349, + "p95": 107.96252799991635, + "max": 350.8312340018165 + } + }, + { + "start": 28770.666981479, + "first": 28770.764170274, + "last": 28776.037162894, + "end": 28776.038667514, + "elapsed_s": 5.371686034999584, + "chunk_offsets_s": [ + 0.09718879499996547, + 0.5431640950009751, + 0.6142942269980267, + 0.6794551569983014, + 0.7396058029989945, + 0.8237414309987798, + 0.914484173998062, + 1.0114840489986818, + 1.1105570280014945, + 1.2061576710002555, + 1.2998992139982875, + 1.3977047400003357, + 1.4997485050007526, + 1.6007842309991247, + 1.700959696998325, + 1.7984896390007634, + 1.9014784290011448, + 1.9992990760001703, + 2.0913577870014706, + 2.1653961619995243, + 2.226891305999743, + 2.3012641749992326, + 2.3804221530008363, + 2.4803340649996244, + 2.580775932998222, + 2.674291772000288, + 2.7380019449992687, + 2.7985113840004487, + 2.875435303998529, + 2.9703278560009494, + 3.052962720001233, + 3.1147144569986267, + 3.18337944599989, + 3.2763835869991453, + 3.374285231999238, + 3.4734693020000122, + 3.5821572019995074, + 3.6931194419994426, + 3.7943496980005875, + 3.8960272459989937, + 3.991698419999011, + 4.088369935001538, + 4.188141459999315, + 4.285009717998037, + 4.382382158000837, + 4.4792725820007036, + 4.566747082000802, + 4.632902393001132, + 4.7107849860003626, + 4.807818463999865, + 4.897029413998098, + 4.967902223997953, + 5.04544057000021, + 5.1313571339997, + 5.20618806299899, + 5.265280681000149, + 5.370181415000843 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 94.31704750022618, + "p95": 108.68789999949513, + "max": 445.9753000010096 + } + }, + { + "start": 28770.666825791, + "first": 28771.05003524, + "last": 28776.036936676, + "end": 28776.03849136, + "elapsed_s": 5.37166556900047, + "chunk_offsets_s": [ + 0.38320944900260656, + 0.5436579140005051, + 0.6143305309997231, + 0.6799234740028623, + 0.7399564650004322, + 0.8240090870021959, + 0.9143073060004099, + 1.0113075170011143, + 1.1112784659999306, + 1.207034218001354, + 1.3005586409999523, + 1.398164698999608, + 1.4994908730004681, + 1.6008408180023252, + 1.7008465280014207, + 1.799129556002299, + 1.901548759000434, + 1.9991943330023787, + 2.09179389700148, + 2.1658276000016485, + 2.2275507300000754, + 2.3018184150023444, + 2.380996990999847, + 2.48086668200267, + 2.5808772270029294, + 2.6747359430009965, + 2.738470751002751, + 2.798989975002769, + 2.8758692080009496, + 2.969879509000748, + 3.053218643002765, + 3.1141871140025614, + 3.184082720999868, + 3.27670669500003, + 3.3745407509995857, + 3.474124913002015, + 3.5818749990030483, + 3.6931291399996553, + 3.794332471999951, + 3.896138456002518, + 3.9921403540029132, + 4.088904926000396, + 4.188731981001183, + 4.285436043999653, + 4.381938722002815, + 4.479790984001738, + 4.567050918001769, + 4.633418168003118, + 4.711083008001879, + 4.80807779099996, + 4.896708899999794, + 4.968118692002463, + 5.045386593003059, + 5.131423568000173, + 5.20681868900283, + 5.265919331002806, + 5.370110885000031 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.9345084989327, + "p95": 107.7500860010332, + "max": 160.44846499789855 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 241.22456915811037, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.381761502998415, + "end_to_end_tps": 237.84034266231527, + "timing_valid": true, + "steady_tps": 256.8520482986068, + "per_stream_tps": [ + 52.2, + 51.2, + 50.2, + 48.2, + 49.1 + ], + "ttft_s": [ + 0.49, + 0.4, + 0.3, + 0.1, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.49152222400152823, + "decode_tps": 52.156184380084994, + "prefill_tps": 28.482944038673764, + "itl_mean_ms": 19.173181701954295, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.96877999947174, + "round_gap_p50_ms": 92.0411539991619, + "round_gap_p95_ms": 105.0579149996338, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3982400950008014, + "decode_tps": 51.176108011019046, + "prefill_tps": 35.15467220841193, + "itl_mean_ms": 19.54036832548274, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.05941900089965, + "round_gap_p50_ms": 94.80655600054888, + "round_gap_p95_ms": 106.86150699984864, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.304569560001255, + "decode_tps": 50.23619030488909, + "prefill_tps": 45.96651090129398, + "itl_mean_ms": 19.90596806666444, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.8462079987803, + "round_gap_p50_ms": 94.78283700082102, + "round_gap_p95_ms": 106.79091899874038, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09550207900247187, + "decode_tps": 48.24611811991009, + "prefill_tps": 146.59366734453644, + "itl_mean_ms": 20.72705616469737, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.53623099863762, + "round_gap_p50_ms": 95.26721099973656, + "round_gap_p95_ms": 107.41661099746125, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18806546599807916, + "decode_tps": 49.10847719429736, + "prefill_tps": 74.44216260386153, + "itl_mean_ms": 20.363083058827232, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.39560499717481, + "round_gap_p50_ms": 94.26993099987158, + "round_gap_p95_ms": 106.501941998431, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.941931463525215, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.92227199836634, + "round_gap_p50_ms": 94.80655600054888, + "round_gap_p95_ms": 105.24468699804856, + "round_gap_mean_ms": 90.80700934283803, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 142.26274414324692 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 28776.040685974, + "first": 28776.532208198, + "last": 28781.421369532, + "end": 28781.421607635, + "elapsed_s": 5.380921660998865, + "chunk_offsets_s": [ + 0.49152222400152823, + 0.5575965890020598, + 0.6249030069993751, + 0.6844582470002933, + 0.7439185799994448, + 0.8301835620004567, + 0.9208436600019922, + 1.0156323260016507, + 1.1153648210020037, + 1.2102300389997254, + 1.310216499001399, + 1.406535141999484, + 1.5085273680015234, + 1.6104938089993084, + 1.7059802809999383, + 1.8009069170002476, + 1.904765850998956, + 2.003046722998988, + 2.094701845999225, + 2.1679200530015805, + 2.2321456230019976, + 2.3088341200018476, + 2.3873013209995406, + 2.485913631000585, + 2.582925221999176, + 2.6713101730019844, + 2.735370024001895, + 2.7959720559993, + 2.8738511960000324, + 2.9658923499991943, + 3.0494165400014026, + 3.1105483760002244, + 3.1787637490015186, + 3.277039993001381, + 3.3770087730008527, + 3.4773445809987606, + 3.586699826999393, + 3.6917577419990266, + 3.7933535990014207, + 3.895479693001107, + 3.9928488030018343, + 4.089359863999562, + 4.187281481001264, + 4.290120540001226, + 4.387027493001369, + 4.4854253430021345, + 4.573559814001783, + 4.634951705000276, + 4.710412213000382, + 4.81113935400208, + 4.901371103998827, + 4.969954209002026, + 5.047630124001444, + 5.13495129399962, + 5.209572140000091, + 5.273233159001393, + 5.380683557999873 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.41490999941016, + "p95": 105.0579149996338, + "max": 109.35524600063218 + } + }, + { + "start": 28776.040255239, + "first": 28776.438495334, + "last": 28781.421289257, + "end": 28781.421411823, + "elapsed_s": 5.381156584000564, + "chunk_offsets_s": [ + 0.3982400950008014, + 0.5581572570008575, + 0.6251739879990055, + 0.6849575609994645, + 0.7444092289988475, + 0.8306862999997975, + 0.9212119199983135, + 1.0160184759988624, + 1.1155264560002252, + 1.2108952009984932, + 1.3105105459981132, + 1.4073140270011208, + 1.5091356129996711, + 1.6105546779981523, + 1.7063333759979287, + 1.8014083880007092, + 1.905144954998832, + 2.0037724469984823, + 2.0949251290003303, + 2.168692954997823, + 2.2329917440001736, + 2.3093067970003176, + 2.3875308169990603, + 2.486063550997642, + 2.5831654159992468, + 2.6715627619996667, + 2.7356405979990086, + 2.7966341949977505, + 2.874325762000808, + 2.966373545001261, + 3.0498876630008454, + 3.110726695998892, + 3.1791670029997476, + 3.277252759999101, + 3.3773121790000005, + 3.4786321520005004, + 3.587079652999819, + 3.6919511310006783, + 3.7941780809997, + 3.8956064070007415, + 3.9927019700007804, + 4.0895902029988065, + 4.187502243999916, + 4.290482879998308, + 4.388110916999722, + 4.485814763000235, + 4.574382629998581, + 4.6352154960004555, + 4.710610795998946, + 4.811385999000777, + 4.9021191920001, + 4.9707258050002565, + 5.048348923999583, + 5.135089111998241, + 5.210262917000364, + 5.274172510999051, + 5.3810340179989 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 94.94078400166472, + "p95": 106.86150699984864, + "max": 159.91716200005612 + } + }, + { + "start": 28776.040428398, + "first": 28776.344997958, + "last": 28781.421019815, + "end": 28781.421922395, + "elapsed_s": 5.3814939969997795, + "chunk_offsets_s": [ + 0.304569560001255, + 0.5582302880029602, + 0.6247326340017025, + 0.6848579610013985, + 0.7446373860002495, + 0.8305495910026366, + 0.9211559430004854, + 1.0159387800013064, + 1.1157656830000633, + 1.210771513000509, + 1.3106056380020163, + 1.4067298750014743, + 1.5087370270011888, + 1.6103029980004067, + 1.706291668000631, + 1.8012787890002073, + 1.904681388001336, + 2.00310166700001, + 2.0946868390019517, + 2.168630457003019, + 2.232705154001451, + 2.3092881940028747, + 2.3870628780023253, + 2.485746772999846, + 2.5833301480015507, + 2.671350271000847, + 2.735424824000802, + 2.795900397002697, + 2.8741925260001153, + 2.9668290530025843, + 3.049403973000153, + 3.110624095003004, + 3.17932149400076, + 3.2773508960017352, + 3.3771971040005155, + 3.4777784400030214, + 3.587101242002973, + 3.6920882300000812, + 3.7938647530027083, + 3.8955201230019156, + 3.9928477890025533, + 4.089682312001969, + 4.187691012000869, + 4.290558671000326, + 4.387634457001695, + 4.486286814000778, + 4.573999995001941, + 4.635463451002579, + 4.710496632000286, + 4.8113781140018546, + 4.901711565002188, + 4.969996325002285, + 5.047980610001105, + 5.135052195000753, + 5.209871975999704, + 5.273800498001947, + 5.380591417000687 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 94.88497900019865, + "p95": 106.79091899874038, + "max": 253.66072800170514 + } + }, + { + "start": 28776.040336197, + "first": 28776.135838276, + "last": 28781.421237598, + "end": 28781.42175712, + "elapsed_s": 5.381420923000405, + "chunk_offsets_s": [ + 0.09550207900247187, + 0.5573846560000675, + 0.6254680320016632, + 0.6845179520023521, + 0.7440557680019992, + 0.8301276310012327, + 0.920827542002371, + 1.0160947530021076, + 1.115301891000854, + 1.2109793260024162, + 1.3105155570010538, + 1.4069423810033186, + 1.508718356002646, + 1.609750547002477, + 1.7058413290033059, + 1.8012331050013017, + 1.9046979360027763, + 2.0029732680013694, + 2.0943493690028845, + 2.168462053003168, + 2.232381184003316, + 2.3089375930030656, + 2.387093007000658, + 2.486380875001487, + 2.5839158530034183, + 2.6720398950019444, + 2.7361879060008505, + 2.7962008460017387, + 2.874137173002964, + 2.9663988250031252, + 3.049919717002922, + 3.111071282000921, + 3.1790457220013195, + 3.2775315760009107, + 3.3768776130018523, + 3.4779925250004453, + 3.5871331980015384, + 3.6919110250019003, + 3.793902571000217, + 3.8959136710000166, + 3.992459962002613, + 4.090010876003362, + 4.1872685630005435, + 4.290707641001063, + 4.387418810001691, + 4.48557579699991, + 4.573865025002306, + 4.635200419001194, + 4.709988605001854, + 4.811236712001119, + 4.901861030000873, + 4.970140297002217, + 5.0480345230025705, + 5.135110223000083, + 5.209871347000444, + 5.27348479000284, + 5.380901401000301 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 95.32949349886621, + "p95": 107.41661099746125, + "max": 461.8825769975956 + } + }, + { + "start": 28776.040160892, + "first": 28776.228226358, + "last": 28781.420812538, + "end": 28781.421052242, + "elapsed_s": 5.380891349999729, + "chunk_offsets_s": [ + 0.18806546599807916, + 0.5579797419995884, + 0.6248272689990699, + 0.6847780429998238, + 0.7447495740016166, + 0.8309291379991919, + 0.9212348239998391, + 1.0165216050008894, + 1.1160906670011173, + 1.2109080320005887, + 1.310830303998955, + 1.4072010759991826, + 1.5092755549994763, + 1.610215723998408, + 1.706950820000202, + 1.8012207510000735, + 1.9051838759987731, + 2.0034384520004096, + 2.094817180000973, + 2.1689544779983407, + 2.2328033379999397, + 2.3090501369988488, + 2.388174864001485, + 2.4865839650010457, + 2.583998273999896, + 2.6721069419982086, + 2.7364254599997366, + 2.7965797069991822, + 2.8747024219992454, + 2.9668392759995186, + 3.0501333749998594, + 3.111303998000949, + 3.179309937000653, + 3.2771978070013574, + 3.3775934119985322, + 3.4785596799993073, + 3.58752362100131, + 3.6927683079993585, + 3.7944079359986063, + 3.8961562010008493, + 3.9930507440003566, + 4.090357624998433, + 4.187749026001256, + 4.2904731380003795, + 4.3880407709984866, + 4.486304602000018, + 4.5745568060010555, + 4.635803852001118, + 4.710828384999331, + 4.8115975159998925, + 4.902314615999785, + 4.970547220000299, + 5.048810650001542, + 5.13500502199895, + 5.20991205899918, + 5.2741497040005925, + 5.3806516459990235 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 94.54364799967152, + "p95": 106.501941998431, + "max": 369.9142760015093 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.5, + "per_stream_tps": 42.5, + "ttft_s_max": 0.11, + "request": { + "ttft_s": 0.086, + "decode_tps": 42.496, + "prefill_tps": null, + "itl_mean_ms": 23.531, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.937, + "round_gap_p50_ms": 59.502, + "round_gap_p95_ms": 78.854, + "tokens_per_chunk": 2.56 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 23.531, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.937, + "round_gap_p50_ms": 59.502, + "round_gap_p95_ms": 78.854, + "tokens_per_chunk": 2.56 + }, + "timing_valid": true, + "end_to_end_tps": 42.07143244650555, + "makespan_s": 6.084889083002054, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.49645830266488, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.084889083002054, + "end_to_end_tps": 42.07143244650555, + "timing_valid": true, + "per_stream_tps": [ + 42.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08407837800041307, + "decode_tps": 42.49645830266488, + "prefill_tps": null, + "itl_mean_ms": 23.531372729413825, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.9667380003084, + "round_gap_p50_ms": 59.362546999182086, + "round_gap_p95_ms": 78.80638300048304, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.531372729413825, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.9667380003084, + "round_gap_p50_ms": 59.362546999182086, + "round_gap_p95_ms": 78.80638300048304, + "round_gap_mean_ms": 60.61111157576288, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28798.156680334, + "first": 28798.240758712, + "last": 28804.241258758, + "end": 28804.241569417, + "elapsed_s": 6.084889083002054, + "chunk_offsets_s": [ + 0.08407837800041307, + 0.14927565900143236, + 0.2200231380011246, + 0.29822386800151435, + 0.35541700000248966, + 0.40335123000113526, + 0.45108613800039166, + 0.502436316000967, + 0.5597473759989953, + 0.6235786680008459, + 0.6878749050010811, + 0.757368888000201, + 0.8284952989997691, + 0.8887593910003488, + 0.9397705699993821, + 0.9959767180007475, + 1.0594845980012906, + 1.1228268570012006, + 1.1857566100006807, + 1.2492958210023062, + 1.307370123002329, + 1.3687749380005698, + 1.4427416760008782, + 1.5236108470016916, + 1.5973084780016507, + 1.6545839620011975, + 1.7084482089994708, + 1.76039221800238, + 1.807713957001397, + 1.852173486000538, + 1.9114843380011735, + 1.976190580000548, + 2.0216983040008927, + 2.0693498239998007, + 2.120536403999722, + 2.17543459800072, + 2.223579197001527, + 2.2845737130010093, + 2.3438992939991294, + 2.400872074002109, + 2.45863713700237, + 2.5072985570004676, + 2.5657135130022652, + 2.628013050001755, + 2.6760372560020187, + 2.735234586001752, + 2.8012142709994805, + 2.8597836930021003, + 2.921879412002454, + 2.9868166710002697, + 3.047616038002161, + 3.096353043001727, + 3.152124503001687, + 3.201926939000259, + 3.2476829450024525, + 3.2939091980006197, + 3.35534134000045, + 3.414221073999215, + 3.4754758950002724, + 3.538716757000657, + 3.5885654279991286, + 3.6505951180006377, + 3.72597015700012, + 3.7948786330016446, + 3.8717226129992923, + 3.94504427100037, + 4.013220045002527, + 4.073710718999791, + 4.16064617200027, + 4.233942183000181, + 4.281426874000317, + 4.326443893001851, + 4.388392749999184, + 4.462675340000715, + 4.526003502000094, + 4.583457418000762, + 4.642819964999944, + 4.69784109900138, + 4.753109352001047, + 4.810366585999873, + 4.856543122001312, + 4.915882381999836, + 4.994688765000319, + 5.072887948001153, + 5.151438230001077, + 5.214750246999756, + 5.274134908999258, + 5.353162429000804, + 5.4319525950013485, + 5.515413812001498, + 5.580371618001664, + 5.632161227000324, + 5.6890811740013305, + 5.745807250001235, + 5.804317874000844, + 5.859432318000472, + 5.908260467000218, + 5.956364838999434, + 6.016783746999863, + 6.084578424000938 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.362546999182086, + "p95": 78.80638300048304, + "max": 86.93545300047845 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.44942992338408, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.113316819002648, + "end_to_end_tps": 41.87579469204819, + "timing_valid": true, + "per_stream_tps": [ + 42.4 + ], + "ttft_s": [ + 0.11 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.10605597000176203, + "decode_tps": 42.44942992338408, + "prefill_tps": null, + "itl_mean_ms": 23.557442392156386, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.74908400015556, + "round_gap_p50_ms": 59.50221899911412, + "round_gap_p95_ms": 79.19952100201044, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.557442392156386, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.74908400015556, + "round_gap_p50_ms": 59.50221899911412, + "round_gap_p95_ms": 79.19952100201044, + "round_gap_mean_ms": 60.67826070706948, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28804.242338308, + "first": 28804.348394278, + "last": 28810.355542088, + "end": 28810.355655127, + "elapsed_s": 6.113316819002648, + "chunk_offsets_s": [ + 0.10605597000176203, + 0.16703169500033255, + 0.2358394680013589, + 0.3103080380024039, + 0.3665938440026366, + 0.41511143100069603, + 0.46257926000180305, + 0.5140586650013574, + 0.5730386400027783, + 0.6394043850013986, + 0.7058393850020366, + 0.7778715200001898, + 0.8487187700011418, + 0.9085203160029778, + 0.9593631429997913, + 1.0149582060003013, + 1.0847592900026939, + 1.1492596330026572, + 1.209051398000156, + 1.2713723350025248, + 1.3285652390004543, + 1.3919547389996296, + 1.4657038229997852, + 1.5449033440017956, + 1.6184498390030058, + 1.677623081002821, + 1.7315339160013536, + 1.7832593680032005, + 1.8305833330014138, + 1.875813406000816, + 1.9347893690028286, + 1.9990087840014894, + 2.0488145710005483, + 2.0984672800004773, + 2.150531965002301, + 2.202170535001642, + 2.2488960650007357, + 2.3085371240013046, + 2.367280059002951, + 2.4243794530011655, + 2.4813973780001106, + 2.5300249780011654, + 2.588483071001974, + 2.6527348900017387, + 2.7005955340027867, + 2.75935091700012, + 2.82584268500068, + 2.884293721002905, + 2.946242052003072, + 3.0109034460001567, + 3.077296039002249, + 3.1291977250002674, + 3.1819754720017954, + 3.229321314000117, + 3.2745868180027173, + 3.3201140529999975, + 3.381543565003085, + 3.4400255780019506, + 3.4995277970010648, + 3.5630007500003558, + 3.614032033001422, + 3.6750216809996346, + 3.7516517490003025, + 3.8197760559996823, + 3.8970186250007828, + 3.9695182730029046, + 4.0396582020002825, + 4.104997993003053, + 4.189657171002182, + 4.260651129003236, + 4.307704062000994, + 4.352633456001058, + 4.414575748000061, + 4.488037516999611, + 4.5518696710023505, + 4.6080466940002225, + 4.666824394000287, + 4.720352971002285, + 4.775343384000735, + 4.832744958999683, + 4.878877607003233, + 4.938574857002095, + 5.017038828002114, + 5.101752444003068, + 5.177649261000624, + 5.237736504001077, + 5.297270366001612, + 5.377568299001723, + 5.456588531000307, + 5.536951379999664, + 5.60188434200245, + 5.655872764000378, + 5.712434638000559, + 5.769942393002566, + 5.827424647002772, + 5.88172935500188, + 5.930701986002532, + 5.97752784900149, + 6.0385079350016895, + 6.1132037800016406 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.50221899911412, + "p95": 79.19952100201044, + "max": 84.71361600095406 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.511165897794534, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.084802515000774, + "end_to_end_tps": 42.0720309934278, + "timing_valid": true, + "per_stream_tps": [ + 42.5 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08628225199936423, + "decode_tps": 42.511165897794534, + "prefill_tps": null, + "itl_mean_ms": 23.52323157648047, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.93723800123553, + "round_gap_p50_ms": 59.824993997608544, + "round_gap_p95_ms": 78.85355399776017, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.52323157648047, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.93723800123553, + "round_gap_p50_ms": 59.824993997608544, + "round_gap_p95_ms": 78.85355399776017, + "round_gap_mean_ms": 60.59014193941939, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28810.356266616, + "first": 28810.442548868, + "last": 28816.44097292, + "end": 28816.441069131, + "elapsed_s": 6.084802515000774, + "chunk_offsets_s": [ + 0.08628225199936423, + 0.14675774100032868, + 0.21350257599988254, + 0.2888897140001063, + 0.3450385110008938, + 0.3927069829987886, + 0.439939421001327, + 0.4929453850018035, + 0.5513467720011249, + 0.6146937189987511, + 0.6811737600000924, + 0.7504530959995463, + 0.8196033380008885, + 0.8800482129990996, + 0.9344170049989771, + 0.9932151530010742, + 1.0585537520018988, + 1.1199079110010643, + 1.180380353001965, + 1.241697531000682, + 1.3015366979998362, + 1.3623865959998511, + 1.4361954880005214, + 1.5142113139991125, + 1.586875229000725, + 1.6463212480011862, + 1.7000110879998829, + 1.7521774730012112, + 1.8006773790002626, + 1.8465325780016428, + 1.9054641729999275, + 1.9749782100006996, + 2.0236494410019077, + 2.072799285000656, + 2.1224414780008374, + 2.173757462998765, + 2.2197868690018367, + 2.2796118629994453, + 2.3372996430007333, + 2.396210298000369, + 2.452708980999887, + 2.5014299559989013, + 2.562181186000089, + 2.625397560001147, + 2.6738389120000647, + 2.732667995001975, + 2.798533311000938, + 2.8561761660021148, + 2.91887882300216, + 2.988219240000035, + 3.051935726998636, + 3.100581653001427, + 3.1520831449997786, + 3.1981393140013097, + 3.243413955999131, + 3.29014706900125, + 3.3501068289988325, + 3.407674395999493, + 3.4673890769990976, + 3.5320886279987462, + 3.5820530240016524, + 3.6456833580014063, + 3.7204921420016035, + 3.7902275790002022, + 3.8666377240006113, + 3.9413281720007944, + 4.012554974000523, + 4.074897685000906, + 4.158063924998714, + 4.227564771001198, + 4.276398732999951, + 4.320829357999173, + 4.382827658999304, + 4.456764897000539, + 4.520166292000795, + 4.577247314999113, + 4.63710208500197, + 4.690566076002142, + 4.744989238999551, + 4.801658921001945, + 4.846775418998732, + 4.906286949000787, + 4.9920011649992375, + 5.072615297001903, + 5.147853661001136, + 5.20748874399942, + 5.269333911001013, + 5.348187464998773, + 5.427303368000139, + 5.505908644001465, + 5.571717226001056, + 5.623819500000536, + 5.68081783299931, + 5.738949044000037, + 5.795923187000881, + 5.852256909001881, + 5.900574629999028, + 5.9527291230006085, + 6.01649230700059, + 6.084706304001884 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.824993997608544, + "p95": 78.85355399776017, + "max": 85.71421599845053 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 80.4, + "per_stream_tps": 40.5, + "ttft_s_max": 0.18, + "request": { + "ttft_s": 0.13, + "decode_tps": 40.544, + "prefill_tps": null, + "itl_mean_ms": 24.665, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.846, + "round_gap_p50_ms": 59.255, + "round_gap_p95_ms": 80.604, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 24.703, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.764, + "round_gap_p50_ms": 59.217, + "round_gap_p95_ms": 80.427, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 79.68601539731424, + "makespan_s": 6.425217743002577, + "steady_tps": 81.3, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 80.43938807568281, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.425217743002577, + "end_to_end_tps": 79.68601539731424, + "timing_valid": true, + "steady_tps": 81.25957456186582, + "per_stream_tps": [ + 40.2, + 40.7 + ], + "ttft_s": [ + 0.08, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08424065000144765, + "decode_tps": 40.21969403784141, + "prefill_tps": null, + "itl_mean_ms": 24.86344125490195, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.94831299906946, + "round_gap_p50_ms": 58.10091400053352, + "round_gap_p95_ms": 80.53517499865848, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16602975000205333, + "decode_tps": 40.744900429944344, + "prefill_tps": null, + "itl_mean_ms": 24.54294867450645, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.07558199783671, + "round_gap_p50_ms": 58.64796100286185, + "round_gap_p95_ms": 80.28056100010872, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.703194964704203, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.94831299906946, + "round_gap_p50_ms": 58.10680500144372, + "round_gap_p95_ms": 80.32760299829533, + "round_gap_mean_ms": 62.36945263365912, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28816.443163132, + "first": 28816.527403782, + "last": 28822.867581302, + "end": 28822.867724609, + "elapsed_s": 6.42456147700068, + "chunk_offsets_s": [ + 0.08424065000144765, + 0.22862648600130342, + 0.2982696789986221, + 0.3770869850013696, + 0.4381591710007342, + 0.4897130540011858, + 0.5414892150001833, + 0.5965419530002691, + 0.6537339759997849, + 0.724105099998269, + 0.7901929180006846, + 0.8695121689997904, + 0.9474604819988599, + 1.0123534880003717, + 1.0677263649995439, + 1.127966234998894, + 1.198452589000226, + 1.264829665000434, + 1.325144727001316, + 1.3919105509994552, + 1.4556159759995353, + 1.5146944980006083, + 1.5916177859980962, + 1.675386738999805, + 1.7531488079985138, + 1.8168296289986756, + 1.8787761999992654, + 1.9337143330012623, + 1.9851348379997944, + 2.033555015001184, + 2.0882624330006365, + 2.153208789000928, + 2.2081334739996237, + 2.257810653998604, + 2.311331875000178, + 2.3653183380010887, + 2.415724800001044, + 2.4709438709978713, + 2.527349601001333, + 2.5848296270014544, + 2.642202267998073, + 2.6928880140003457, + 2.7467735149984946, + 2.804874428999028, + 2.8585669009989942, + 2.914583608999237, + 2.9710407739985385, + 3.025365180001245, + 3.08659205100048, + 3.1538101219994132, + 3.235541916001239, + 3.2870653690006293, + 3.3395902949996525, + 3.391132983000716, + 3.4398433399983332, + 3.488699532997998, + 3.542890090000583, + 3.5987653900010628, + 3.6519406590014114, + 3.7120294599990302, + 3.7641037169996707, + 3.822013801000139, + 3.9025489759987977, + 3.9782708150014514, + 4.059811415998411, + 4.138364297999942, + 4.211319063000701, + 4.267697444000078, + 4.341582891000144, + 4.419070873998862, + 4.492461343998002, + 4.539915662000567, + 4.5976378209998074, + 4.666289512999356, + 4.733575591999397, + 4.78972137099845, + 4.85135292799896, + 4.9195319389982615, + 4.976490961998934, + 5.031876453998848, + 5.091806348998944, + 5.138300417998835, + 5.1933964300005755, + 5.262792016997992, + 5.342315771998983, + 5.422344931001135, + 5.4869968829989375, + 5.542763141998876, + 5.610848124000768, + 5.6895433659992705, + 5.77452498799903, + 5.8449740879987075, + 5.905774054997892, + 5.969322981000005, + 6.027246737998212, + 6.082772838999517, + 6.140368455999123, + 6.193138618000376, + 6.241884609000408, + 6.296816830999887, + 6.352563608998025, + 6.424418170001445 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.10091400053352, + "p95": 80.53517499865848, + "max": 144.38583599985577 + } + }, + { + "start": 28816.442934065, + "first": 28816.608963815, + "last": 28822.867415727, + "end": 28822.868151808, + "elapsed_s": 6.425217743002577, + "chunk_offsets_s": [ + 0.16602975000205333, + 0.22867440000118222, + 0.2991143870021915, + 0.3771899690000282, + 0.43881342399981804, + 0.49016817700248794, + 0.541473383000266, + 0.5969721040019067, + 0.6540562930022134, + 0.7244164510011615, + 0.7905022840022866, + 0.8698175010031264, + 0.9479546500006109, + 1.0122664500013343, + 1.0677411100004974, + 1.1279569670005003, + 1.1987627070011513, + 1.265228518001095, + 1.3250866170019435, + 1.3919317960026092, + 1.455681164003181, + 1.5149706299998797, + 1.5919791150008678, + 1.6751112180027121, + 1.7535021010007767, + 1.816765868003131, + 1.878722141002072, + 1.934125160001713, + 1.9854671120010607, + 2.033563517001312, + 2.088171057999716, + 2.15322870500313, + 2.2082389230017725, + 2.2581096950016217, + 2.3116504610006814, + 2.365435890002118, + 2.415891034001106, + 2.471100323000428, + 2.5274762460030615, + 2.5848808880000433, + 2.6422551620016748, + 2.692950272001326, + 2.7468922840016603, + 2.804999089003104, + 2.859009457002685, + 2.9150314159996924, + 2.9714887410009396, + 3.0255253620016447, + 3.08671551000225, + 3.1539436029997887, + 3.2356163740005286, + 3.2872288010003103, + 3.3401157580010477, + 3.3911976760027756, + 3.440003073999833, + 3.489120766000269, + 3.5429398300002504, + 3.598933417000808, + 3.651972278003086, + 3.7121683830009715, + 3.763965823000035, + 3.822613784002897, + 3.9028943450030056, + 3.9782675679998647, + 4.0597691720031435, + 4.139163298001222, + 4.211781901001814, + 4.267570195002918, + 4.342139699001564, + 4.419659250001132, + 4.492939181000111, + 4.54044810400228, + 4.5982003589997476, + 4.666625804002251, + 4.733879418003198, + 4.789862936002464, + 4.851464141000179, + 4.919532743002492, + 4.97652563900192, + 5.032203716000367, + 5.091830492001463, + 5.138247417002276, + 5.193317319000926, + 5.262813999001082, + 5.342659011002979, + 5.422986614001275, + 5.487138768003206, + 5.542744095000671, + 5.6109540540019225, + 5.689634537000529, + 5.7749455949997355, + 5.845121112000925, + 5.906332366001152, + 5.96944839400021, + 6.027393506003136, + 6.082896843003255, + 6.140382307003165, + 6.193124422003166, + 6.242384346001927, + 6.296820835999824, + 6.352510128999711, + 6.424481662001199 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.64796100286185, + "p95": 80.28056100010872, + "max": 85.31105799920624 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 80.36746569508838, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.430906402998517, + "end_to_end_tps": 79.61552663264878, + "timing_valid": true, + "steady_tps": 81.19482854136605, + "per_stream_tps": [ + 40.2, + 40.7 + ], + "ttft_s": [ + 0.08, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08404407800117042, + "decode_tps": 40.18428855747624, + "prefill_tps": null, + "itl_mean_ms": 24.88534787842974, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.51600199824315, + "round_gap_p50_ms": 59.37689200072782, + "round_gap_p95_ms": 81.53962500000489, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16674721500021406, + "decode_tps": 40.71186528646267, + "prefill_tps": null, + "itl_mean_ms": 24.562863749023936, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.52162499976112, + "round_gap_p50_ms": 59.184907000599196, + "round_gap_p95_ms": 81.02667300045141, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.72410581372684, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.76434400086873, + "round_gap_p50_ms": 59.21738499819185, + "round_gap_p95_ms": 81.23774199702893, + "round_gap_mean_ms": 62.42224735148856, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28822.869594513, + "first": 28822.953638591, + "last": 28829.2994023, + "end": 28829.300118989, + "elapsed_s": 6.430524475999846, + "chunk_offsets_s": [ + 0.08404407800117042, + 0.22863080900060595, + 0.2969782260006468, + 0.37532709000151954, + 0.43850572800147347, + 0.4924701140007528, + 0.5444484720028413, + 0.5998168240002997, + 0.6590443910026806, + 0.7283301860006759, + 0.7949049340022611, + 0.8714209360005043, + 0.946185280001373, + 1.0115324950020295, + 1.0660460520011839, + 1.1270963570023014, + 1.1961010450031608, + 1.2624290740022843, + 1.3205682000007073, + 1.3875150800013216, + 1.4546765990016866, + 1.5168578299999353, + 1.5909530880016973, + 1.6753138690000924, + 1.7526755810031318, + 1.8156657120016462, + 1.8725583000014012, + 1.9280793420002738, + 1.9777812410029583, + 2.0262045380004565, + 2.080417340002896, + 2.143522168000345, + 2.1983616150027956, + 2.2479512780009827, + 2.3005648810030834, + 2.355560697000328, + 2.4045466870011296, + 2.465552153000317, + 2.5250370370013115, + 2.5800937800013344, + 2.639470672002062, + 2.690547508002055, + 2.7438972370000556, + 2.8038749050028855, + 2.8539465150024625, + 2.907424359000288, + 2.9625013440017938, + 3.017560101001436, + 3.0784449330021744, + 3.1464054960015346, + 3.219152699002734, + 3.2718285890005063, + 3.3245678880011837, + 3.3752660650025064, + 3.425538998002594, + 3.4772162420013046, + 3.5336661680012185, + 3.5886497980027343, + 3.642668800002866, + 3.7046850510014337, + 3.7571129240022856, + 3.81549373800226, + 3.890125951002119, + 3.9636208950032596, + 4.046734223000385, + 4.125480732000142, + 4.199217594999936, + 4.2564204370028165, + 4.328968286001327, + 4.406150883001828, + 4.472734574999777, + 4.52322066400302, + 4.580151573001785, + 4.649602925001091, + 4.717471602001751, + 4.775294460003352, + 4.833968719001859, + 4.8958344840029895, + 4.950538685003266, + 5.006855744002678, + 5.0670156340020185, + 5.113852649003093, + 5.168774140001915, + 5.23989125099979, + 5.321127680002974, + 5.3996643320024305, + 5.470468403000268, + 5.529681548003282, + 5.598335901002429, + 5.679875526002434, + 5.765138497001317, + 5.834678523002367, + 5.890698276001785, + 5.952915096000652, + 6.013461123002344, + 6.068282379001175, + 6.125251252000453, + 6.177745549000974, + 6.228770722002082, + 6.281909527002426, + 6.338005343000987, + 6.429807787000755 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.37689200072782, + "p95": 81.53962500000489, + "max": 144.58673099943553 + } + }, + { + "start": 28822.869212586, + "first": 28823.035959801, + "last": 28829.299490057, + "end": 28829.299671782, + "elapsed_s": 6.430459195998992, + "chunk_offsets_s": [ + 0.16674721500021406, + 0.2286355790020025, + 0.29721633400185965, + 0.3756081159990572, + 0.4391294609995384, + 0.4930486240009486, + 0.5450763519984321, + 0.6001119220018154, + 0.6593293070000072, + 0.7288091400005214, + 0.7953667260007933, + 0.8719696219995967, + 0.9464912469993578, + 1.0115958560018043, + 1.06635253100103, + 1.1275617829996918, + 1.1961174050011323, + 1.262952626999322, + 1.3211034220003057, + 1.3884484779991908, + 1.4551423299999442, + 1.5170020650002698, + 1.5912267829990014, + 1.6753516489989124, + 1.752767756999674, + 1.815808683000796, + 1.872730726001464, + 1.928389248001622, + 1.978401403001044, + 2.026712339000369, + 2.0807165680016624, + 2.143545252001786, + 2.1988186050002696, + 2.2482089490004, + 2.301139195998985, + 2.355875452998589, + 2.4051984979996632, + 2.466223830000672, + 2.525342651999381, + 2.580813914999453, + 2.639720514001965, + 2.690843231001054, + 2.7442084229987813, + 2.8041263800005254, + 2.853918420998525, + 2.907606854001642, + 2.962670024000545, + 3.0176652279988048, + 3.0787600570001814, + 3.146669985999324, + 3.219369757000095, + 3.2720106529995974, + 3.324671351001598, + 3.3755323799996404, + 3.425715923000098, + 3.477829456998734, + 3.5335673980007414, + 3.5888281000006828, + 3.6432406659987464, + 3.7052804960003414, + 3.7573465339992254, + 3.8157736290013418, + 3.890119980998861, + 3.9641002009993827, + 4.04671549500199, + 4.125620134000201, + 4.19948949800164, + 4.256411106001906, + 4.3292764789985085, + 4.406752555001731, + 4.472887054998864, + 4.523404537001625, + 4.58066340699952, + 4.649750827000389, + 4.717740814001445, + 4.77576697800032, + 4.834258549999504, + 4.896043713000836, + 4.951156989998708, + 5.007444465001754, + 5.0671944320019975, + 5.114484033001645, + 5.169387498001015, + 5.240040354001394, + 5.321278095998423, + 5.4002794199986965, + 5.471091941999475, + 5.530276849000074, + 5.5989778180010035, + 5.680004491001455, + 5.765583175998472, + 5.835124483000982, + 5.891379381999286, + 5.953451182002027, + 6.013902425002016, + 6.068400585998461, + 6.125704016001691, + 6.178193558000203, + 6.228946527000517, + 6.282066042000224, + 6.338436703001207, + 6.430277471001318 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.184907000599196, + "p95": 81.02667300045141, + "max": 91.84076800011098 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 80.7469896539852, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.409451445000741, + "end_to_end_tps": 79.8820311525023, + "timing_valid": true, + "steady_tps": 81.60262816631013, + "per_stream_tps": [ + 40.9, + 40.4 + ], + "ttft_s": [ + 0.18, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.17641528099920833, + "decode_tps": 40.91381243304334, + "prefill_tps": null, + "itl_mean_ms": 24.441623513734623, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.26184200085117, + "round_gap_p50_ms": 59.32563700116589, + "round_gap_p95_ms": 79.71518000340438, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.09311877000072855, + "decode_tps": 40.37651423806166, + "prefill_tps": null, + "itl_mean_ms": 24.76687299215472, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.17651799935265, + "round_gap_p50_ms": 59.45128500025021, + "round_gap_p95_ms": 80.67210099761724, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.60424825294467, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.42915400315542, + "round_gap_p50_ms": 59.422825997899054, + "round_gap_p95_ms": 80.42674399985117, + "round_gap_mean_ms": 62.11963667822665, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28829.301195639, + "first": 28829.47761092, + "last": 28835.710224916, + "end": 28835.710532657, + "elapsed_s": 6.409337018001679, + "chunk_offsets_s": [ + 0.17641528099920833, + 0.2393121679997421, + 0.3079887179992511, + 0.3876757069992891, + 0.44743577799818013, + 0.49754390599991893, + 0.5486114679988532, + 0.6033306030003587, + 0.6615315589988313, + 0.7315973990007478, + 0.7965797610013396, + 0.870644472001004, + 0.9442241659999127, + 1.0121362259997113, + 1.0705030120006995, + 1.1301273880017106, + 1.1988231309987896, + 1.2648285149989533, + 1.3246121410011256, + 1.392208039000252, + 1.4551365459992667, + 1.5144621830004326, + 1.5917168330015556, + 1.6735176549991593, + 1.7509914570000547, + 1.812753005000559, + 1.8683173569988867, + 1.9232535459996143, + 1.9732391789984831, + 2.0264593840001908, + 2.082741623999027, + 2.1458458449997124, + 2.2001721849992464, + 2.2494573629992374, + 2.3015291550000256, + 2.358842707999429, + 2.408849632000056, + 2.4647401830006856, + 2.521608469000057, + 2.5779121609994036, + 2.636953720000747, + 2.6884209659983753, + 2.7419423690007534, + 2.8006531179999, + 2.8499260880016664, + 2.902165650000825, + 2.9576487459999043, + 3.016740186001698, + 3.0788455219990283, + 3.145779763999599, + 3.218242791001103, + 3.2692215950009995, + 3.3220962019986473, + 3.373318810998171, + 3.4202430580007785, + 3.468638698999712, + 3.5224313639992033, + 3.577520246999484, + 3.6308157480016234, + 3.6930438019990106, + 3.7464474869993865, + 3.8038990499990177, + 3.879222105999361, + 3.95304900499832, + 4.038971259000391, + 4.119977735001157, + 4.193918707998819, + 4.250500097998156, + 4.324761939999007, + 4.4014580990005925, + 4.462673714999255, + 4.5090125360002276, + 4.566607378001208, + 4.634852155999397, + 4.701980972000456, + 4.7570854219993635, + 4.816755708001438, + 4.879649591999623, + 4.934436311999889, + 4.990339217998553, + 5.057450135998806, + 5.106365376999747, + 5.160471860999678, + 5.23065604599833, + 5.311082789998181, + 5.390797970001586, + 5.45418196099854, + 5.508790862000751, + 5.578786581998429, + 5.65687550200164, + 5.741511217998777, + 5.810139632001665, + 5.865600450000784, + 5.930256118001125, + 5.999013317999925, + 6.061540187998617, + 6.122682047000126, + 6.1781420490005985, + 6.227424674998474, + 6.2817118860002665, + 6.339025616998697, + 6.409029277001537 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.32563700116589, + "p95": 79.71518000340438, + "max": 85.92225400207099 + } + }, + { + "start": 28829.301081212, + "first": 28829.394199982, + "last": 28835.709752595, + "end": 28835.709939443, + "elapsed_s": 6.408858230999613, + "chunk_offsets_s": [ + 0.09311877000072855, + 0.23920998699759366, + 0.3082138419995317, + 0.3878685099989525, + 0.4477944749996823, + 0.49777469699984067, + 0.5486419640001259, + 0.6031808459993044, + 0.6615634320005483, + 0.7318996979993244, + 0.7965097889973549, + 0.8709389430005103, + 0.9441672719985945, + 1.0123265479996917, + 1.0703246800003399, + 1.1300257999973837, + 1.1986711649988138, + 1.265194783998595, + 1.3248448369995458, + 1.392098846998124, + 1.4550093129982997, + 1.51446059799855, + 1.5917847079981584, + 1.673975473997416, + 1.751210385999002, + 1.8126796879987523, + 1.868659148000006, + 1.923458612000104, + 1.9734740359999705, + 2.0268008719976933, + 2.0830651500000386, + 2.1457169949972013, + 2.2003745129986783, + 2.249780488997203, + 2.301400192998699, + 2.3591948889989, + 2.4087983099998382, + 2.4646016959995904, + 2.5216448079991096, + 2.5779483719998098, + 2.637371197997709, + 2.6883444630002487, + 2.7419171879992064, + 2.8005074279972177, + 2.8498728449994815, + 2.902077204998932, + 2.95800646600037, + 3.0166431609977735, + 3.0788372789975256, + 3.145619105998776, + 3.2181465339999704, + 3.269133277997753, + 3.3219732599973213, + 3.373509708999336, + 3.420418860998325, + 3.468597413997486, + 3.5224014289997285, + 3.5774825640000927, + 3.6306544020008005, + 3.6928715219983133, + 3.7462807939991762, + 3.803907342000457, + 3.8790838599998096, + 3.9530442989998846, + 4.039156458999059, + 4.120210990997293, + 4.1941776249987015, + 4.250807503998658, + 4.325040514999273, + 4.401728286000434, + 4.462933496997721, + 4.509305453997513, + 4.566876395998406, + 4.635111136998603, + 4.702254264000658, + 4.757367357997282, + 4.817027878998488, + 4.8799398689989175, + 4.9346921229989675, + 4.990672428997641, + 5.057243199000368, + 5.106209569999919, + 5.160394205999182, + 5.230692832999921, + 5.311364933997538, + 5.390699103998486, + 5.454134048999549, + 5.5090993880003225, + 5.578748146999715, + 5.656844110999373, + 5.741843739997421, + 5.8103593769992585, + 5.865854307998234, + 5.930617582998821, + 5.998983222998504, + 6.06187640799908, + 6.122544137997465, + 6.177998584000306, + 6.227310586000385, + 6.281578330999764, + 6.338927599997987, + 6.408671383000183 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.45128500025021, + "p95": 80.67210099761724, + "max": 146.0912169968651 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 137.9, + "per_stream_tps": 35.0, + "ttft_s_max": 0.41, + "request": { + "ttft_s": 0.212, + "decode_tps": 35.064, + "prefill_tps": null, + "itl_mean_ms": 28.52, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.825, + "round_gap_p50_ms": 66.667, + "round_gap_p95_ms": 96.125, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 28.522, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.407, + "round_gap_p50_ms": 66.77, + "round_gap_p95_ms": 96.449, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 136.6826869609958, + "makespan_s": 7.491804724999383, + "steady_tps": 141.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 135.7382310294325, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.5999645619995135, + "end_to_end_tps": 134.73747037191325, + "timing_valid": true, + "steady_tps": 141.105237113887, + "per_stream_tps": [ + 35.1, + 35.5, + 34.3, + 33.9 + ], + "ttft_s": [ + 0.33, + 0.41, + 0.17, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.3322830370016163, + "decode_tps": 35.095279939468305, + "prefill_tps": null, + "itl_mean_ms": 28.493860192162067, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.4536490017781, + "round_gap_p50_ms": 68.5905520003871, + "round_gap_p95_ms": 97.7274180004315, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.4137617289998161, + "decode_tps": 35.49551124851287, + "prefill_tps": null, + "itl_mean_ms": 28.172576329405487, + "itl_p50_ms": 0.0, + "itl_p95_ms": 90.8958630025154, + "round_gap_p50_ms": 67.69443300072453, + "round_gap_p95_ms": 96.20770499896025, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17056316399975913, + "decode_tps": 34.32543631751811, + "prefill_tps": null, + "itl_mean_ms": 29.13291445882208, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.22601399940322, + "round_gap_p50_ms": 68.23835800241795, + "round_gap_p95_ms": 97.68333200190682, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08537312499902328, + "decode_tps": 33.939241774778836, + "prefill_tps": null, + "itl_mean_ms": 29.464417815695782, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.45259800061467, + "round_gap_p50_ms": 68.40224800180295, + "round_gap_p95_ms": 97.28220100078033, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 28.815942199021354, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.22601399940322, + "round_gap_p50_ms": 68.12460299988743, + "round_gap_p95_ms": 97.7274180004315, + "round_gap_mean_ms": 72.75312139356876, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28835.712849407, + "first": 28836.045132444, + "last": 28843.311066793, + "end": 28843.311561508, + "elapsed_s": 7.598712101000274, + "chunk_offsets_s": [ + 0.3322830370016163, + 0.4807698500007973, + 0.5573545620027289, + 0.6533570050014532, + 0.7224512930006313, + 0.7768238080025185, + 0.8322758499998599, + 0.9060476560007373, + 0.9776473160018213, + 1.0568310790004034, + 1.12739646800037, + 1.2153666539998085, + 1.3076631540025119, + 1.3907508550000784, + 1.4516989560033835, + 1.5157440560033137, + 1.5931811710033799, + 1.6754813360021217, + 1.7405495650018565, + 1.8195959950026008, + 1.9023661589999392, + 1.9740472570010752, + 2.0657884500033106, + 2.170033278001938, + 2.256788189002691, + 2.330410873000801, + 2.394051651001064, + 2.4545429730023898, + 2.509268708003219, + 2.560423348000768, + 2.62445728700186, + 2.6984423950016208, + 2.769895864003047, + 2.8292422040030942, + 2.8871281530009583, + 2.949114475002716, + 3.00476112600154, + 3.0668294580027577, + 3.1308552980008244, + 3.1995560830000613, + 3.2754648810005165, + 3.3324696640011098, + 3.389923267000995, + 3.45630076100133, + 3.510232721000648, + 3.56690030900063, + 3.6318486130003294, + 3.6934594150006888, + 3.758121670001856, + 3.8459258830007457, + 3.9341207910001685, + 3.9898401540012856, + 4.047785430000658, + 4.105368123000517, + 4.157594598000287, + 4.209274316002848, + 4.266965243001323, + 4.3272851670008095, + 4.396826408999914, + 4.476427089000936, + 4.532616919001157, + 4.598444469000242, + 4.692585980999866, + 4.775919707000867, + 4.877292278000823, + 4.978361528003006, + 5.062450309000269, + 5.125983855003142, + 5.2076185530022485, + 5.293115928001498, + 5.361041775002377, + 5.412889938001172, + 5.477246728001774, + 5.552766295000765, + 5.640803469003004, + 5.713346471002296, + 5.7821624650023296, + 5.854575258003024, + 5.916576390001865, + 5.9788943540006585, + 6.047484906001046, + 6.100014939001994, + 6.15858160100106, + 6.250876076002896, + 6.346811403000174, + 6.438265052001952, + 6.514219446002244, + 6.577575637002155, + 6.657723963002354, + 6.749013193002611, + 6.846740611003042, + 6.924165014002938, + 6.988064888002555, + 7.058687030003057, + 7.123248787000193, + 7.1827699080022285, + 7.248571998999978, + 7.30837523000082, + 7.363196213002084, + 7.421281375001854, + 7.4813507740000205, + 7.598217386002943 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.5905520003871, + "p95": 97.7274180004315, + "max": 148.48681299918098 + } + }, + { + "start": 28835.713340842, + "first": 28836.127102571, + "last": 28843.311109535, + "end": 28843.311814372, + "elapsed_s": 7.598473529997136, + "chunk_offsets_s": [ + 0.4137617289998161, + 0.48034523299793364, + 0.5567929789976915, + 0.6527176389972738, + 0.7220037839979341, + 0.7762218690004374, + 0.8318279889972473, + 0.9053538939988357, + 0.9772373619998689, + 1.0560384989985323, + 1.1271867279974686, + 1.2149692669991055, + 1.307086986998911, + 1.389902502996847, + 1.450916669000435, + 1.5149915279980632, + 1.5928839789994527, + 1.6751179339989903, + 1.7399698449989955, + 1.8185654529988824, + 1.9021169269981328, + 1.9733501970004, + 2.0655289409987745, + 2.1694954829981725, + 2.2562124709984346, + 2.3302145959969494, + 2.393738339000265, + 2.454151717000059, + 2.5089782389986794, + 2.5600620429977425, + 2.6241371229989454, + 2.69786461199692, + 2.7688490910004475, + 2.828632597000251, + 2.8860765939971316, + 2.948231048998423, + 3.004043319997436, + 3.0662021930002084, + 3.1302478839970718, + 3.1991217009999673, + 3.2752680589983356, + 3.33203464199687, + 3.3895649240002967, + 3.455929739997373, + 3.5095014039980015, + 3.566467158998421, + 3.631639464998443, + 3.6931005109981925, + 3.7573879199990188, + 3.8451159029973496, + 3.9338896319968626, + 3.989484050998726, + 4.047630021999794, + 4.105185468997661, + 4.157198603999859, + 4.209021929997107, + 4.2667638909988455, + 4.326671429997077, + 4.396383878996858, + 4.47543604099701, + 4.532191676997172, + 4.597743711998191, + 4.692045032999886, + 4.775494944999082, + 4.876882788998046, + 4.9778026189997036, + 5.062203429999499, + 5.125237026997638, + 5.2073631579987705, + 5.292714458999399, + 5.360408892000123, + 5.412198737998551, + 5.476263507996919, + 5.552750075999938, + 5.640529151998635, + 5.71326412399867, + 5.781388726998557, + 5.853848006998305, + 5.9158327789991745, + 5.978082804998849, + 6.047043799997482, + 6.099610411998583, + 6.158313718999125, + 6.25009097200018, + 6.34629867699914, + 6.4378219619975425, + 6.5139783619997615, + 6.577017960997182, + 6.657337508997443, + 6.748233371999959, + 6.846192459997837, + 6.923894346000452, + 6.9875035299992305, + 7.057783385997027, + 7.123129588999291, + 7.18234365799799, + 7.248176675999275, + 7.308018935000291, + 7.36295187900032, + 7.420963099997607, + 7.480914790998213, + 7.597768692998216 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 67.69443300072453, + "p95": 96.20770499896025, + "max": 116.85390200000256 + } + }, + { + "start": 28835.712523112, + "first": 28835.883086276, + "last": 28843.311979463, + "end": 28843.312107272, + "elapsed_s": 7.599584159997903, + "chunk_offsets_s": [ + 0.17056316399975913, + 0.48129306199916755, + 0.5578644220004207, + 0.6536302809981862, + 0.7228733960000682, + 0.7773017159997835, + 0.8328436439987854, + 0.906901835998724, + 0.9782402429991635, + 1.0571031699982996, + 1.127670591999049, + 1.21549913799754, + 1.3084032910010137, + 1.3909610429982422, + 1.4519909289992938, + 1.5160096319996228, + 1.5936171219982498, + 1.6765837380007724, + 1.7409946560001117, + 1.81950101899929, + 1.9021712289977586, + 1.9743222939978295, + 2.0662966769996274, + 2.170427639997797, + 2.256876266998006, + 2.3308153029975074, + 2.3943245749978814, + 2.454739442000573, + 2.509681670999271, + 2.5608044549990154, + 2.6250845510003273, + 2.6991162579979573, + 2.770057771998836, + 2.829009558998223, + 2.8876003789991955, + 2.949339037997561, + 3.0052166870009387, + 3.067104318000929, + 3.131474925998191, + 3.199713284000609, + 3.2762499050004408, + 3.3324401629979548, + 3.3902870370002347, + 3.4566763449984137, + 3.510252219999529, + 3.5671527899976354, + 3.6326399440004025, + 3.6941181990005134, + 3.757992197999556, + 3.8458671829976083, + 3.9345768479979597, + 3.990110340000683, + 4.048309234000044, + 4.105838570998458, + 4.157800400000269, + 4.209674392001034, + 4.267415153000911, + 4.327561997000885, + 4.397370525999577, + 4.476592871000321, + 4.533128251998278, + 4.5984512100003485, + 4.693146762998367, + 4.776175405997492, + 4.87780812499841, + 4.978560782998102, + 5.062933082997915, + 5.126471399998991, + 5.208247832997586, + 5.294011088000843, + 5.36129327999879, + 5.413609647999692, + 5.477399382998556, + 5.5532272339987685, + 5.641457705998619, + 5.713828845000535, + 5.7823582620003435, + 5.854775151998183, + 5.916699350000272, + 5.979028391997417, + 6.047647470000811, + 6.099888395998278, + 6.159421131000272, + 6.251086919997761, + 6.347208581999439, + 6.438434595998842, + 6.5143454799981555, + 6.577665716999036, + 6.658390493998013, + 6.749263048997818, + 6.846946380999725, + 6.92494047999935, + 6.988207857997622, + 7.058938150999893, + 7.124154799999815, + 7.183349659000669, + 7.249082130998431, + 7.30915677899975, + 7.363719086999481, + 7.421461804999126, + 7.481934159000957, + 7.5994563509993895 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.23835800241795, + "p95": 97.68333200190682, + "max": 310.7298979994084 + } + }, + { + "start": 28835.71214271, + "first": 28835.797515835, + "last": 28843.310942378, + "end": 28843.311228859, + "elapsed_s": 7.599086148999049, + "chunk_offsets_s": [ + 0.08537312499902328, + 0.48190084399902844, + 0.5582085820024076, + 0.6541014809990884, + 0.723320072000206, + 0.7776496059996134, + 0.8331009909998102, + 0.9071373799997673, + 0.9782653600013873, + 1.057373932999326, + 1.1280164010022418, + 1.2160289280000143, + 1.3088965189999726, + 1.3911788990008063, + 1.4523179280004115, + 1.5163383440012694, + 1.594346804999077, + 1.6767398180018063, + 1.7414412680009264, + 1.8200810590024048, + 1.9024136879997968, + 1.97485269200115, + 2.0663052900017647, + 2.1706007040011173, + 2.2572119739998016, + 2.3310697529996105, + 2.3946545030012203, + 2.4551900070000556, + 2.509851726001216, + 2.5610197970017907, + 2.625018678001652, + 2.6993754630020703, + 2.770251279001968, + 2.829503683002258, + 2.887444684001821, + 2.9495916470004886, + 3.005358215999877, + 3.067776373001834, + 3.1316770289995475, + 3.2001508030007244, + 3.276309520002542, + 3.332985783999902, + 3.3905731990016648, + 3.4568756640001084, + 3.5105265519996465, + 3.5674171009995916, + 3.6326934280004934, + 3.6940639640015434, + 3.758478350002406, + 3.8463776190001226, + 3.93490022900005, + 3.9904302150025615, + 4.048640539000189, + 4.106106084000203, + 4.1581123510004545, + 4.209934381000494, + 4.267702852001094, + 4.327804601001844, + 4.397403504000977, + 4.47655853499964, + 4.5335938970019924, + 4.5990567980006745, + 4.693071189001785, + 4.7764933120015485, + 4.8778459380009735, + 4.978883283001778, + 5.063603456001147, + 5.126534473001811, + 5.2087871479998284, + 5.294162974001665, + 5.361950014001195, + 5.413827055999718, + 5.477563210999506, + 5.553662832000555, + 5.641361098001653, + 5.714265099999466, + 5.782667348001269, + 5.85509189000004, + 5.917136964999372, + 5.979323982999631, + 6.048081339002238, + 6.100493328001903, + 6.159142046999477, + 6.251157533999503, + 6.347258896999847, + 6.438893262002239, + 6.514629993002018, + 6.57795437599998, + 6.6588849860017945, + 6.749816035000549, + 6.8470982360013295, + 6.925271593001526, + 6.988541372000327, + 7.059087924000778, + 7.124076714000694, + 7.183637775000534, + 7.2495071470002586, + 7.3093324700021185, + 7.364040899999964, + 7.421934573001636, + 7.482374063001771, + 7.598799668001448 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.40224800180295, + "p95": 97.28220100078033, + "max": 396.52771900000516 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 137.8539910096865, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.491804724999383, + "end_to_end_tps": 136.6826869609958, + "timing_valid": true, + "steady_tps": 141.85289642668982, + "per_stream_tps": [ + 35.7, + 34.5, + 35.3, + 34.9 + ], + "ttft_s": [ + 0.34, + 0.09, + 0.26, + 0.18 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.34461293799904524, + "decode_tps": 35.68189236307491, + "prefill_tps": null, + "itl_mean_ms": 28.025419443135842, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.12958000271465, + "round_gap_p50_ms": 66.36939799864194, + "round_gap_p95_ms": 95.57728500294616, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.09208482800022466, + "decode_tps": 34.464868259449595, + "prefill_tps": null, + "itl_mean_ms": 29.015053603920844, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.68153499861364, + "round_gap_p50_ms": 66.21688900122535, + "round_gap_p95_ms": 96.63752200140152, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25933155200254987, + "decode_tps": 35.2620721981809, + "prefill_tps": null, + "itl_mean_ms": 28.35908208626457, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.67362800083356, + "round_gap_p50_ms": 66.23632300033933, + "round_gap_p95_ms": 96.0425330013095, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17570524400071008, + "decode_tps": 34.85901975253739, + "prefill_tps": null, + "itl_mean_ms": 28.686979929411525, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.33238299848745, + "round_gap_p50_ms": 66.27205299810157, + "round_gap_p95_ms": 96.44894799930626, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 28.521633765683195, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.40704500107677, + "round_gap_p50_ms": 66.23632300033933, + "round_gap_p95_ms": 96.44894799930626, + "round_gap_mean_ms": 72.01006544801203, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28843.31387948, + "first": 28843.658492418, + "last": 28850.804974376, + "end": 28850.8053882, + "elapsed_s": 7.491508719998819, + "chunk_offsets_s": [ + 0.34461293799904524, + 0.4124379240020062, + 0.4897841359997983, + 0.5808323080018454, + 0.6478300330018101, + 0.7032885010012251, + 0.7583806080001523, + 0.8216412760011735, + 0.8853394940015278, + 0.9652196040005947, + 1.0423323309987609, + 1.1309141040001123, + 1.2134820530009165, + 1.2890949809989252, + 1.3503904800018063, + 1.4152094240016595, + 1.4899906300015573, + 1.5677866190017085, + 1.6309436439987621, + 1.7091706819992396, + 1.7798958179992042, + 1.8453184840000176, + 1.9329660250004963, + 2.029348153999308, + 2.1181856240000343, + 2.1903282450002735, + 2.2559038840008725, + 2.318037431999983, + 2.372288229998958, + 2.423405925001134, + 2.481206768999982, + 2.552229502998671, + 2.6117969830011134, + 2.6661103699989326, + 2.724144307001552, + 2.7858725740006776, + 2.8420125630000257, + 2.9043279899997287, + 2.967648324000038, + 3.0364542600000277, + 3.1032380380020186, + 3.1589667799999006, + 3.217296834998706, + 3.282504554001207, + 3.336251271000947, + 3.393752811000013, + 3.4522887140010425, + 3.5110724580008537, + 3.5763013670002692, + 3.650345860001835, + 3.7352133160020458, + 3.792035056001623, + 3.8502867169991077, + 3.909731859999738, + 3.9606623709987616, + 4.018404350001219, + 4.0812598379998235, + 4.141119013998832, + 4.201648144000501, + 4.270049405000464, + 4.329747635998501, + 4.394207019999158, + 4.480944121998618, + 4.567221384000732, + 4.661435016001633, + 4.751485781001975, + 4.836125066001841, + 4.899831139999151, + 4.9816195470011735, + 5.0782399119998445, + 5.157238512998447, + 5.216765502998896, + 5.282450205999339, + 5.354113059998781, + 5.429012601998693, + 5.49297593600204, + 5.559345334000682, + 5.629846365998674, + 5.7010068019990285, + 5.7724007479992, + 5.840144886999042, + 5.89129314299862, + 5.949147769999399, + 6.033831420998467, + 6.1294087060014135, + 6.228228168998612, + 6.316357749001327, + 6.379596113001753, + 6.453688730998692, + 6.542731968998851, + 6.641115810001793, + 6.725852473999112, + 6.800218762000441, + 6.870437810001022, + 6.935588784999709, + 6.99575566600106, + 7.065334278999217, + 7.126968346001377, + 7.180732450000505, + 7.243473426999117, + 7.316002822000883, + 7.491094895998685 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.36939799864194, + "p95": 95.57728500294616, + "max": 175.0920739978028 + } + }, + { + "start": 28843.31375665, + "first": 28843.405841478, + "last": 28850.804680147, + "end": 28850.804991168, + "elapsed_s": 7.491234517998237, + "chunk_offsets_s": [ + 0.09208482800022466, + 0.41215256299983594, + 0.49012899800072773, + 0.5808963079980458, + 0.6480641509988345, + 0.7034586349982419, + 0.7584148329988238, + 0.8215961969981436, + 0.8851756349977222, + 0.96523535599772, + 1.0423024279989477, + 1.1311202079996292, + 1.2139359779976076, + 1.2890202390008199, + 1.3504384080006275, + 1.4151854119991185, + 1.4900589530006982, + 1.5679550719978579, + 1.6311130089998187, + 1.709205450999434, + 1.7803536809988145, + 1.8458658319977985, + 1.9328786369987938, + 2.0295161590001953, + 2.118197693998809, + 2.1903798389976146, + 2.2556549740002083, + 2.3175932390004164, + 2.3719787289992382, + 2.4230784949977533, + 2.4812694729989744, + 2.552093579000939, + 2.6114132849979796, + 2.665914798999438, + 2.7239995780000754, + 2.786220541998773, + 2.842192141997657, + 2.9044041720007954, + 2.9678795449981408, + 3.0366568099998403, + 3.1035585999998148, + 3.159285309000552, + 3.217442459997983, + 3.2826858769985847, + 3.336438484999235, + 3.3933905480007525, + 3.4527465289975225, + 3.511492477999127, + 3.5762292189974687, + 3.650321190998511, + 3.735049089998938, + 3.792111062000913, + 3.8503705670009367, + 3.9099800969997887, + 3.9606948670007114, + 4.018261397999595, + 4.081005100997572, + 4.141338564997568, + 4.201688979999744, + 4.27025089900053, + 4.33011470099882, + 4.394454777000647, + 4.481100888999208, + 4.567503062000469, + 4.6614928689996304, + 4.751711542998237, + 4.836300577997463, + 4.900055924998014, + 4.981629951998912, + 5.078274553998199, + 5.157278580998536, + 5.216801040998689, + 5.2819593509993865, + 5.354132509997726, + 5.429032627998822, + 5.492875948999426, + 5.559092838000652, + 5.6300629709985515, + 5.701076084998931, + 5.772689565998007, + 5.8406293709995225, + 5.891468142999656, + 5.949182427000778, + 6.033857033999084, + 6.129592045999743, + 6.227948651998304, + 6.3166051209991565, + 6.37961823599835, + 6.453766689999611, + 6.54314348799744, + 6.64039890499771, + 6.725412732997938, + 6.800398900999426, + 6.870491580997623, + 6.93538932500087, + 6.995517513998493, + 7.065784169000835, + 7.12695110599816, + 7.180806534997828, + 7.243681931999163, + 7.31607586699829, + 7.49092349700004 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.21688900122535, + "p95": 96.63752200140152, + "max": 320.0677349996113 + } + }, + { + "start": 28843.313680916, + "first": 28843.573012468, + "last": 28850.8045784, + "end": 28850.805136361, + "elapsed_s": 7.491455445000611, + "chunk_offsets_s": [ + 0.25933155200254987, + 0.4123878339996736, + 0.4899348840008315, + 0.5810892050030816, + 0.6480768460023683, + 0.7034279630024685, + 0.758636929000204, + 0.8218918350030435, + 0.8853417990030721, + 0.9653736649997882, + 1.0422903569997288, + 1.1309639850005624, + 1.2139111650030827, + 1.2892415509995772, + 1.3503090789999987, + 1.4151223240005493, + 1.4902324320028129, + 1.5678704530000687, + 1.6313904609996825, + 1.7094251300004544, + 1.7801829390009516, + 1.8457621800007473, + 1.9334294750005938, + 2.0294720080019033, + 2.1183439119995455, + 2.190812681001262, + 2.2560363020020304, + 2.318171915001585, + 2.3720054940022237, + 2.423691845000576, + 2.48168046399951, + 2.552303797001514, + 2.6116054779995466, + 2.6663669789995765, + 2.7245253319997573, + 2.7865724630028126, + 2.8421557390029193, + 2.904345615002967, + 2.9678093969996553, + 3.036400056000275, + 3.1034937190015626, + 3.1592879980016733, + 3.21743679300198, + 3.2826476660011394, + 3.3365966450000997, + 3.393638417001057, + 3.452968769001018, + 3.5111915100023907, + 3.576404971001466, + 3.650351958000101, + 3.7350168970006052, + 3.792402328999742, + 3.850538860002416, + 3.909790273002727, + 3.9609597850030696, + 4.01839388100052, + 4.0814039899996715, + 4.141367939002521, + 4.201952233001066, + 4.270286596001824, + 4.330017083000712, + 4.394348484001966, + 4.480875271001423, + 4.567274291002832, + 4.661515023999527, + 4.751733026001602, + 4.836622645001626, + 4.899912701002904, + 4.981991830001789, + 5.078482914999768, + 5.15752196700123, + 5.216714805002994, + 5.282447101002617, + 5.354594342999917, + 5.429245423001703, + 5.492885874002241, + 5.55912219700258, + 5.630099613001221, + 5.70124417000261, + 5.772641317002126, + 5.840553237001586, + 5.891443602002255, + 5.949431673001527, + 6.033779987003072, + 6.129767141999764, + 6.228167994002433, + 6.316742934999638, + 6.3798430530005135, + 6.454025572002138, + 6.543240369002888, + 6.64093176499955, + 6.725709218000702, + 6.800875332999567, + 6.870952149001823, + 6.935940242001379, + 6.995764822000638, + 7.06573568100066, + 7.127115316001436, + 7.181156520000513, + 7.243913825001073, + 7.316405969002517, + 7.4908974840000155 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.23632300033933, + "p95": 96.0425330013095, + "max": 174.4915149974986 + } + }, + { + "start": 28843.313583475, + "first": 28843.489288719, + "last": 28850.804468601, + "end": 28850.804730365, + "elapsed_s": 7.491146889999072, + "chunk_offsets_s": [ + 0.17570524400071008, + 0.41240840400132583, + 0.4899602400000731, + 0.58057470499989, + 0.6483010070005548, + 0.7037811980007973, + 0.7588122020024457, + 0.8218804690004617, + 0.8855660240005818, + 0.9655974900015281, + 1.042586987001414, + 1.1309940320024907, + 1.213980160002393, + 1.289120641999034, + 1.350566200002504, + 1.415315925001778, + 1.4903607849992113, + 1.5680189760023495, + 1.6311995809992368, + 1.709328937002283, + 1.780332726000779, + 1.8455665450019296, + 1.933318858002167, + 2.029767806001473, + 2.1181001889999607, + 2.1907187610013352, + 2.256063756001822, + 2.3180150040025183, + 2.3723804040018877, + 2.4234437350023654, + 2.481880500999978, + 2.5520291140019253, + 2.612325681002403, + 2.6662599650007905, + 2.7241259460024594, + 2.786518420001812, + 2.842423633999715, + 2.9044916080019902, + 2.968123779999587, + 3.036552229001245, + 3.1036770589998923, + 3.159303173000808, + 3.2176425130019197, + 3.2826035309990402, + 3.3366516160022, + 3.3937837070006935, + 3.4526993690014933, + 3.511603846000071, + 3.576560858000448, + 3.6506045820024156, + 3.7354675560018222, + 3.7924395160007407, + 3.850445773001411, + 3.9097942110020085, + 3.9610080330021447, + 4.018337468001846, + 4.080981601000531, + 4.141560867999942, + 4.201899949999643, + 4.2704664949997095, + 4.329966114000854, + 4.394570467000449, + 4.481026835001103, + 4.56746412999928, + 4.661786889002542, + 4.751961396999832, + 4.8366840349990525, + 4.90007028400214, + 4.98213422200206, + 5.078613027999381, + 5.1578672290015675, + 5.216754073000629, + 5.2821965260009165, + 5.354449934002332, + 5.429607476999081, + 5.493117351001274, + 5.559389403999376, + 5.630519041002117, + 5.70152703200074, + 5.772787263002101, + 5.84030384700236, + 5.89173983100045, + 5.949483619002422, + 6.033971876000578, + 6.129645289001928, + 6.228806588002044, + 6.31671623299917, + 6.379987590000383, + 6.453882236000936, + 6.543257801000436, + 6.641197242999624, + 6.726002548999531, + 6.800764445000823, + 6.870784401002311, + 6.935678286001348, + 6.9962713490022, + 7.065571533999901, + 7.1274720239998715, + 7.181188214999565, + 7.243916290000925, + 7.316588380999747, + 7.490885126000649 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.27205299810157, + "p95": 96.44894799930626, + "max": 236.70316000061575 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 138.56369421910767, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.44563206499879, + "end_to_end_tps": 137.53029844352997, + "timing_valid": true, + "steady_tps": 142.5282552925017, + "per_stream_tps": [ + 35.0, + 35.4, + 35.9, + 34.6 + ], + "ttft_s": [ + 0.17, + 0.25, + 0.33, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.1660007050013519, + "decode_tps": 35.03180344004752, + "prefill_tps": null, + "itl_mean_ms": 28.545490149011968, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.22392900037812, + "round_gap_p50_ms": 66.77002599826665, + "round_gap_p95_ms": 94.27607000179705, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.24897494199831272, + "decode_tps": 35.44058137046063, + "prefill_tps": null, + "itl_mean_ms": 28.21624141960295, + "itl_p50_ms": 0.0, + "itl_p95_ms": 89.04367000286584, + "round_gap_p50_ms": 67.05965299988748, + "round_gap_p95_ms": 93.65984100077185, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33201064899913035, + "decode_tps": 35.85381120221357, + "prefill_tps": null, + "itl_mean_ms": 27.89103770196294, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.31995600101072, + "round_gap_p50_ms": 66.56352600111859, + "round_gap_p95_ms": 93.32581699709408, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08412705600130721, + "decode_tps": 34.646250764232576, + "prefill_tps": null, + "itl_mean_ms": 28.863151941172248, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.96888099843636, + "round_gap_p50_ms": 66.53283099876717, + "round_gap_p95_ms": 93.56023999862373, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 28.378980302937528, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.46918299948447, + "round_gap_p50_ms": 66.77002599826665, + "round_gap_p95_ms": 93.65984100077185, + "round_gap_mean_ms": 71.64990076484227, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28850.807089953, + "first": 28850.973090658, + "last": 28858.252190646, + "end": 28858.252460046, + "elapsed_s": 7.445370092998928, + "chunk_offsets_s": [ + 0.1660007050013519, + 0.398357648999081, + 0.485296136001125, + 0.5876635429995076, + 0.6604090299988457, + 0.7158759769990866, + 0.7696591330022784, + 0.8342130720011482, + 0.8978992930024106, + 0.9776315130002331, + 1.0486820980004268, + 1.1355452480020176, + 1.2223764289992687, + 1.3112708890002978, + 1.3738325459999032, + 1.4391725130008126, + 1.517066636999516, + 1.5989788799997768, + 1.6649960030008515, + 1.7428870539988566, + 1.819902881001326, + 1.894407764000789, + 1.9826316930011672, + 2.0754749680017994, + 2.161938346001989, + 2.2356103620004433, + 2.298850762999791, + 2.3588062950002495, + 2.42281802800062, + 2.482857122999121, + 2.5465221220001695, + 2.617586537999159, + 2.6772663480005576, + 2.730948644999444, + 2.7870637020023423, + 2.8506592260018806, + 2.9064902040008747, + 2.9740307680003752, + 3.0484565410006326, + 3.1162019459989097, + 3.182353750002221, + 3.23879339099949, + 3.297367433002364, + 3.3637442940016626, + 3.41937085400059, + 3.477071961002366, + 3.5477580610022414, + 3.6174099769996246, + 3.6885548509999353, + 3.7633104209999146, + 3.846941013001924, + 3.9021201159994234, + 3.9616634769990924, + 4.018925332002254, + 4.078617317001772, + 4.1407148630023585, + 4.200171192998823, + 4.26043738700173, + 4.318327154000144, + 4.38855462599895, + 4.445588522001344, + 4.5119573030024185, + 4.605334361000132, + 4.689493075002247, + 4.7837921009995625, + 4.876832109999668, + 4.960885395001242, + 5.024195382000471, + 5.107095865001611, + 5.192044639999949, + 5.2619953520006675, + 5.313716569999087, + 5.3784389440006635, + 5.451936558001762, + 5.532633053000609, + 5.599403078998876, + 5.66419707300156, + 5.734560296001291, + 5.795821747000446, + 5.859571072000108, + 5.928379722001409, + 5.980728343001829, + 6.040341611998883, + 6.11965013200097, + 6.211816679002368, + 6.3042235060020175, + 6.3804600200019195, + 6.4444400899992615, + 6.521617028000037, + 6.615893098001834, + 6.714243533999252, + 6.791398493001907, + 6.856624885000201, + 6.927335327000037, + 6.991997897999681, + 7.052393278001546, + 7.117268427002273, + 7.176856649999536, + 7.231401732999075, + 7.289847301999544, + 7.350622841000586, + 7.445100692999404 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.77002599826665, + "p95": 94.27607000179705, + "max": 232.3569439977291 + } + }, + { + "start": 28850.80716106, + "first": 28851.056136002, + "last": 28858.251277564, + "end": 28858.251501308, + "elapsed_s": 7.444340247999207, + "chunk_offsets_s": [ + 0.24897494199831272, + 0.3981659529999888, + 0.48547625999708544, + 0.5873189869998896, + 0.6601606169970182, + 0.7160192659976019, + 0.7697024519984552, + 0.8342478739978105, + 0.8980008649996307, + 0.9774584829974629, + 1.0488278679986252, + 1.1356827259987767, + 1.2220823299976473, + 1.3112845119976555, + 1.3736366079974687, + 1.4389364269991347, + 1.5173257599999488, + 1.598814893997769, + 1.6648006419964076, + 1.7429234409974015, + 1.8193604969965236, + 1.893837124996935, + 1.9828807949998009, + 2.0757002659993304, + 2.161660702997324, + 2.2353329269972164, + 2.298602798997308, + 2.3590093099992373, + 2.42315757599863, + 2.482593070999428, + 2.5461834559973795, + 2.6170855189993745, + 2.676821516997734, + 2.7306355289983912, + 2.78682990499874, + 2.8503627579993918, + 2.906590270999004, + 2.97430255299696, + 3.0482310359984695, + 3.115953181997611, + 3.1816592419963854, + 3.2387805369980924, + 3.296726470998692, + 3.3637861239985796, + 3.4191751559992554, + 3.47735283899965, + 3.548139372996957, + 3.617633929996373, + 3.68774567699802, + 3.7632128529985494, + 3.846483455999987, + 3.9021034679972217, + 3.961069739998493, + 4.018922291998024, + 4.078392132996669, + 4.141179160997126, + 4.1997347669966985, + 4.260530377996474, + 4.318057674998272, + 4.38836757199897, + 4.444966545997886, + 4.511739769997803, + 4.605399610998575, + 4.6895070989994565, + 4.7840419549975195, + 4.876487137997174, + 4.960638327997003, + 5.0239722469996195, + 5.106864612997015, + 5.191816253998695, + 5.261828468999738, + 5.313448083998082, + 5.378153408997605, + 5.451632117998088, + 5.5327143589966, + 5.5985682449972956, + 5.663887381997483, + 5.73435039999822, + 5.7955839639980695, + 5.8593415489995095, + 5.928076866999618, + 5.98043014599898, + 6.040014311998675, + 6.119313211998815, + 6.211506827999983, + 6.303648017998057, + 6.380581168999925, + 6.444571691998135, + 6.521304614998371, + 6.615602728998056, + 6.713970597997104, + 6.791268860997661, + 6.856593525997596, + 6.927449784998316, + 6.992080275998887, + 7.052386778999789, + 7.11733614199693, + 7.177163011998346, + 7.231545741997252, + 7.29010173399729, + 7.350745334999374, + 7.444116503997066 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 67.05965299988748, + "p95": 93.65984100077185, + "max": 149.1910110016761 + } + }, + { + "start": 28850.806932593, + "first": 28851.138943242, + "last": 28858.251157856, + "end": 28858.251826495, + "elapsed_s": 7.444893902000331, + "chunk_offsets_s": [ + 0.33201064899913035, + 0.39857417500024894, + 0.4858444450001116, + 0.5873346909975226, + 0.6602578979982354, + 0.7159743640004308, + 0.7697228939978231, + 0.8343023819979862, + 0.8981926739979826, + 0.9777333569982147, + 1.0489232119980443, + 1.136243167999055, + 1.222359637999034, + 1.3115603949991055, + 1.3739148919994477, + 1.4390753769985167, + 1.517332515999442, + 1.5994733209990954, + 1.665120098998159, + 1.742802099997789, + 1.8203119190002326, + 1.894284597998194, + 1.9832273699976213, + 2.0757859730001655, + 2.1618045830000483, + 2.235465056997782, + 2.298701632000302, + 2.35903722799776, + 2.4231951309993747, + 2.482747500998812, + 2.5467377989989473, + 2.617502159999276, + 2.677200652000465, + 2.7310262369974225, + 2.7871126879981603, + 2.85076227099853, + 2.9064924140002404, + 2.974626619998162, + 3.0484067409997806, + 3.116128326000762, + 3.1823695989987755, + 3.238526393997745, + 3.2970902220004064, + 3.36417915399943, + 3.4196650669982773, + 3.4771358979996876, + 3.5484940959977394, + 3.6177095519997238, + 3.6878465119989414, + 3.763582958999905, + 3.8468993290007347, + 3.902368225997634, + 3.9620029129982868, + 4.0194007879981655, + 4.078457301999151, + 4.140931964997435, + 4.200280480999936, + 4.260698046997277, + 4.3183630450002966, + 4.388415580997389, + 4.445664929997292, + 4.51202607400046, + 4.605351890997554, + 4.689538635000645, + 4.7844187849987065, + 4.877177660997404, + 4.960808813000767, + 5.024246512999525, + 5.107001337997644, + 5.192099404997862, + 5.262015731997963, + 5.313636307000706, + 5.378424712998822, + 5.451903021999897, + 5.532994691999193, + 5.599314091999986, + 5.664160510998045, + 5.7345219739981985, + 5.795766616000037, + 5.859496570999909, + 5.9283538540003065, + 5.980985193000379, + 6.040329349998501, + 6.119731150000007, + 6.211772193000797, + 6.304268152998702, + 6.3810320679986035, + 6.444084631999431, + 6.521961793998344, + 6.616305913998076, + 6.714323110998521, + 6.791673127998365, + 6.8569872439984465, + 6.92763844000001, + 6.9924214870006836, + 7.052805325998634, + 7.1176203489994805, + 7.177207914999599, + 7.231383131998882, + 7.289813028997742, + 7.350659915999131, + 7.44422526299968 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.56352600111859, + "p95": 93.32581699709408, + "max": 101.49024599741097 + } + }, + { + "start": 28850.806827981, + "first": 28850.890955037, + "last": 28858.251058782, + "end": 28858.252314164, + "elapsed_s": 7.445486183001776, + "chunk_offsets_s": [ + 0.08412705600130721, + 0.39841377300035674, + 0.48586515900024096, + 0.5877393580012722, + 0.6604118630020821, + 0.7164659869995376, + 0.7701220870003453, + 0.834538787999918, + 0.8982511989997874, + 0.9777211100008572, + 1.0489938870014157, + 1.1364630700009002, + 1.222326052000426, + 1.3112949329988624, + 1.3739046939990658, + 1.4392168880003737, + 1.5175933350001287, + 1.5994603219987766, + 1.6650737550007761, + 1.743000728001789, + 1.8199085000014747, + 1.8940000389993656, + 1.9835612970018701, + 2.076086987999588, + 2.1617789060001087, + 2.2360798619993147, + 2.298901989001024, + 2.3587998600014544, + 2.422916494000674, + 2.4829916070011677, + 2.546649129999423, + 2.617516087000695, + 2.6772481789994345, + 2.7310377940011676, + 2.7872598329995526, + 2.850808085000608, + 2.9066523980000056, + 2.9743946950002282, + 3.048274176999257, + 3.1163723680001567, + 3.1822392450012558, + 3.2388065500017547, + 3.2977610390007612, + 3.3642322039995634, + 3.419425042000512, + 3.4775342419998196, + 3.548220790002233, + 3.617906595001841, + 3.688508028000797, + 3.7634407750010723, + 3.846722343001602, + 3.90225194300001, + 3.961706555001001, + 4.01909407399944, + 4.078375548000622, + 4.1408163539999805, + 4.200248320001265, + 4.261077277002187, + 4.3182575030004955, + 4.388597510998807, + 4.445489369001734, + 4.5120222000005015, + 4.60552738700062, + 4.689531990999967, + 4.784257983999851, + 4.876982681998925, + 4.961069791999762, + 5.0247093370016955, + 5.1072472050000215, + 5.192366226001468, + 5.262318779001362, + 5.313872201000777, + 5.378623453001637, + 5.452101024999138, + 5.532984366000164, + 5.599018679000437, + 5.664356897999824, + 5.734888957998919, + 5.796369366998988, + 5.859722043998772, + 5.928550159998849, + 5.980879394999647, + 6.040489974999218, + 6.119754905001173, + 6.211964177000482, + 6.3040884309994, + 6.380577455001912, + 6.444427010999789, + 6.5217553049988055, + 6.616290530000697, + 6.714549463998992, + 6.79171601300186, + 6.856965104001574, + 6.92753235499913, + 6.992308134002087, + 7.052968030999182, + 7.117606758001784, + 7.176984330999403, + 7.232017450001877, + 7.289996913001232, + 7.350670561001607, + 7.444230801000231 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.53283099876717, + "p95": 93.56023999862373, + "max": 314.28671699904953 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 160.8, + "per_stream_tps": 32.8, + "ttft_s_max": 0.51, + "request": { + "ttft_s": 0.271, + "decode_tps": 32.819, + "prefill_tps": null, + "itl_mean_ms": 30.47, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.665, + "round_gap_p50_ms": 71.509, + "round_gap_p95_ms": 103.174, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 30.412, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.765, + "round_gap_p50_ms": 71.509, + "round_gap_p95_ms": 103.653, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 159.62058036409124, + "makespan_s": 8.019016075999389, + "steady_tps": 166.9, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 160.8223637417629, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 8.019016075999389, + "end_to_end_tps": 159.62058036409124, + "timing_valid": true, + "steady_tps": 166.93861247251417, + "per_stream_tps": [ + 33.2, + 32.2, + 32.9, + 33.6, + 32.5 + ], + "ttft_s": [ + 0.35, + 0.09, + 0.26, + 0.43, + 0.18 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.34780354099711985, + "decode_tps": 33.24523702309954, + "prefill_tps": null, + "itl_mean_ms": 30.07949678040128, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.50986600111355, + "round_gap_p50_ms": 70.83932100067614, + "round_gap_p95_ms": 103.0091739994532, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08979899200130603, + "decode_tps": 32.167916146871605, + "prefill_tps": null, + "itl_mean_ms": 31.086875364702543, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.33524699832196, + "round_gap_p50_ms": 70.58514600066701, + "round_gap_p95_ms": 103.65546599859954, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.26319704500201624, + "decode_tps": 32.883923991116276, + "prefill_tps": null, + "itl_mean_ms": 30.409996090191484, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.28453700072714, + "round_gap_p50_ms": 71.03836500027683, + "round_gap_p95_ms": 103.65258399906452, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.43132256399985636, + "decode_tps": 33.61721104640161, + "prefill_tps": null, + "itl_mean_ms": 29.74666752157717, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.87801099708304, + "round_gap_p50_ms": 70.4680750022817, + "round_gap_p95_ms": 102.372725999885, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17987751500186278, + "decode_tps": 32.532361565081736, + "prefill_tps": null, + "itl_mean_ms": 30.738623078422297, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.95824600121705, + "round_gap_p50_ms": 71.4156419999199, + "round_gap_p95_ms": 103.17434399985359, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 30.412331767058955, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.28453700072714, + "round_gap_p50_ms": 70.83932100067614, + "round_gap_p95_ms": 103.65258399906452, + "round_gap_mean_ms": 76.78360990693102, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28858.254531694, + "first": 28858.602335235, + "last": 28866.272606914, + "end": 28866.272787885, + "elapsed_s": 8.018256190996908, + "chunk_offsets_s": [ + 0.34780354099711985, + 0.5014045030002308, + 0.5845277829976112, + 0.6773442279991286, + 0.7479291460003878, + 0.8061560209971503, + 0.8629822879993299, + 0.9281481979996897, + 0.992551420997188, + 1.0807944499974838, + 1.1620763379978598, + 1.2534553890000097, + 1.3384309219982242, + 1.4198114509999868, + 1.4851865219970932, + 1.5524127589997079, + 1.6359702569970977, + 1.7177756909986783, + 1.7863167260002228, + 1.8695466989993292, + 1.9464340019985684, + 2.0120753409973986, + 2.108335517998057, + 2.2104517679981655, + 2.2998846949994913, + 2.379264367998985, + 2.447355896998488, + 2.5101246909980546, + 2.565619830998912, + 2.621113142999093, + 2.681513747997087, + 2.755382710998674, + 2.8217148529984115, + 2.87739747500018, + 2.9368177649994323, + 3.003421642999456, + 3.067078378997394, + 3.139577501999156, + 3.212679167998431, + 3.2809681789985916, + 3.3540273750004417, + 3.414943759999005, + 3.475747799999226, + 3.5485133749971283, + 3.6062196459970437, + 3.665799528997013, + 3.733597197999188, + 3.8070317249985237, + 3.8777068210001744, + 3.95518014299887, + 4.047596734999388, + 4.113309630000003, + 4.176353505998122, + 4.238451260000147, + 4.292677561999881, + 4.346960211998521, + 4.407198552999034, + 4.471594594997441, + 4.542433915998117, + 4.629590452997945, + 4.688337094998133, + 4.755586256997049, + 4.847191519998887, + 4.937301985999511, + 5.0563823859993136, + 5.15907719399911, + 5.252587060000224, + 5.3201215009976295, + 5.412188755999523, + 5.515197929998976, + 5.593274673999986, + 5.646385140997154, + 5.714420369997242, + 5.794605641996895, + 5.884620193999581, + 5.959182855000108, + 6.028945557998668, + 6.120738841000275, + 6.193400309999561, + 6.262839825998526, + 6.338668049997068, + 6.3927928489974875, + 6.45412655799737, + 6.549819107000076, + 6.656262253000023, + 6.751781058999768, + 6.837313194999297, + 6.904092538999976, + 6.9837422229975346, + 7.078722966998612, + 7.203060842999548, + 7.296483125999657, + 7.365837952998845, + 7.441342310998152, + 7.510812395998073, + 7.576137215997733, + 7.662176269997872, + 7.728808718999062, + 7.7867202529996575, + 7.847270197999023, + 7.913142008997966, + 8.018075219999446 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.83932100067614, + "p95": 103.0091739994532, + "max": 153.6009620031109 + } + }, + { + "start": 28858.254806055, + "first": 28858.344605047, + "last": 28866.271758265, + "end": 28866.271865567, + "elapsed_s": 8.017059511999832, + "chunk_offsets_s": [ + 0.08979899200130603, + 0.501339927999652, + 0.5844464149995474, + 0.6772523110012116, + 0.7477070509994519, + 0.8056895310000982, + 0.8628558740019798, + 0.9277492149994941, + 0.9922237690007023, + 1.080601410001691, + 1.161545560000377, + 1.2532554489989707, + 1.3382217620019219, + 1.4194744660017022, + 1.485090266000043, + 1.5522096489985415, + 1.6356292220007163, + 1.7174449019985332, + 1.7859361189985066, + 1.8693278540013125, + 1.9455572430015309, + 2.0120109419985965, + 2.107916363998811, + 2.2102457619985216, + 2.299720132999937, + 2.378451673001109, + 2.446856910999486, + 2.5095756000009715, + 2.564997166999092, + 2.620415608998883, + 2.681117518000974, + 2.754903559001832, + 2.820894346001296, + 2.8769400470009714, + 2.9364228960002947, + 3.003038907998416, + 3.0667230969993398, + 3.139448525998887, + 3.2125530090015673, + 3.28077605100043, + 3.3538441640012024, + 3.4147442520006734, + 3.4755301229997713, + 3.548545648001891, + 3.6060005930012267, + 3.665219974998763, + 3.7330073340017407, + 3.80735995500072, + 3.8775661590007076, + 3.955047004001244, + 4.047541316002025, + 4.113126514999749, + 4.176015786000789, + 4.238287610998668, + 4.292536564000329, + 4.346852910999587, + 4.407065590999991, + 4.471397809000337, + 4.541982955001004, + 4.629451102999155, + 4.68817211699934, + 4.755514268999832, + 4.846994558000006, + 4.93723481699999, + 5.056388678000076, + 5.159085567000147, + 5.252377627999522, + 5.319959708998795, + 5.411849034000625, + 5.515504499999224, + 5.593127289001131, + 5.646372223000071, + 5.7143093869999575, + 5.794566647000465, + 5.8841717450013675, + 5.9587898579993634, + 6.028776705999917, + 6.120371601999068, + 6.193207910000638, + 6.263087998999254, + 6.338927141001477, + 6.3929889960018045, + 6.453308197000297, + 6.549235855000006, + 6.655526491998899, + 6.751183143001981, + 6.836859240000194, + 6.90360451900051, + 6.983511835998797, + 7.078673535001144, + 7.202938446000189, + 7.296273692998511, + 7.365417647000868, + 7.4408253010005865, + 7.510783967001771, + 7.576099148998765, + 7.6615364300014335, + 7.728268768998532, + 7.786143549001281, + 7.847522291998757, + 7.912298834999092, + 8.016952210000454 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.58514600066701, + "p95": 103.65546599859954, + "max": 411.54093599834596 + } + }, + { + "start": 28858.254430635, + "first": 28858.51762768, + "last": 28866.272176683, + "end": 28866.272227204, + "elapsed_s": 8.017796569001803, + "chunk_offsets_s": [ + 0.26319704500201624, + 0.5012950570016983, + 0.5844598780022352, + 0.6773209849998238, + 0.7479162599993288, + 0.8058661800023401, + 0.8628426030008995, + 0.9279101590000209, + 0.9923991370014846, + 1.0811272259998077, + 1.1619781450026494, + 1.2538221009999688, + 1.3389368880016264, + 1.4202994560000661, + 1.4859076840002672, + 1.552303712000139, + 1.635807343001943, + 1.7176074630006042, + 1.7860614300007, + 1.8694673960017099, + 1.9466689680011768, + 2.012701128001936, + 2.108095381001476, + 2.2103813489993627, + 2.29972043700036, + 2.3796025219999137, + 2.4476024370014784, + 2.5098945280005864, + 2.565849721999257, + 2.621462966002582, + 2.6816961920012545, + 2.755393069001002, + 2.821381453999493, + 2.8778022549995512, + 2.9370892780025315, + 3.003819203000603, + 3.0675407710004947, + 3.139976807000494, + 3.213253680001799, + 3.281301049999456, + 3.3545258500016644, + 3.4154627400021127, + 3.476117938000243, + 3.5487220079994586, + 3.606205318999855, + 3.6652964610002527, + 3.7337776720014517, + 3.8079313289999845, + 3.878251783000451, + 3.95517534100145, + 4.047561949999363, + 4.1132503369990445, + 4.176315248001629, + 4.238463859001058, + 4.292691312999523, + 4.3469800780003425, + 4.40723439499925, + 4.471569814999384, + 4.542608179999661, + 4.63005775900092, + 4.688276840999606, + 4.755606860002445, + 4.847010678000515, + 4.937324508999154, + 5.0571238299999095, + 5.159282144999452, + 5.252566682000179, + 5.320162449999771, + 5.412375618001533, + 5.516028202000598, + 5.593550060999405, + 5.646374160001869, + 5.7144782390023465, + 5.794654483001068, + 5.885073044999444, + 5.959340566001629, + 6.029435996002576, + 6.121382124001684, + 6.193386767001357, + 6.2636315199997625, + 6.338868183000159, + 6.392755263001163, + 6.45385535099922, + 6.549708284001099, + 6.6563964640008635, + 6.751968049000425, + 6.837494245999551, + 6.904159724999772, + 6.98373732500113, + 7.079382642001292, + 7.2034482050003135, + 7.2962453110012575, + 7.365390898001351, + 7.440927512001508, + 7.510999338999682, + 7.576929277001909, + 7.661707843999466, + 7.728762232000008, + 7.786683547001303, + 7.846831130002101, + 7.912855786002183, + 8.017746048000845 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 71.03836500027683, + "p95": 103.65258399906452, + "max": 238.09801199968206 + } + }, + { + "start": 28858.25539267, + "first": 28858.686715234, + "last": 28866.272115452, + "end": 28866.273336624, + "elapsed_s": 8.017943954000657, + "chunk_offsets_s": [ + 0.43132256399985636, + 0.5004577240033541, + 0.5835576330027834, + 0.6764356439998664, + 0.7469037190021481, + 0.8049546180009202, + 0.8619253100005153, + 0.9269944190018578, + 0.9914724640002532, + 1.0803752800020447, + 1.1614370890019927, + 1.2529253470020194, + 1.3377067190012895, + 1.419508979000966, + 1.4851293890023953, + 1.5514491070025542, + 1.6347990770009346, + 1.7167181850018096, + 1.7850017780001508, + 1.8690556210021896, + 1.945130052001332, + 2.0113095010019606, + 2.1070741010007623, + 2.2093674000025203, + 2.2987245930016798, + 2.3782365400002163, + 2.4465688780001074, + 2.5088148020004155, + 2.5645013820030726, + 2.619990179002343, + 2.680912839001394, + 2.7549225760012632, + 2.82071164500303, + 2.8767227370008186, + 2.936287019001611, + 3.0030452140017587, + 3.0664368890029436, + 3.138941295001132, + 3.212539193002158, + 3.2808284450002247, + 3.353641982001136, + 3.4143286990001798, + 3.475193730002502, + 3.547706378001749, + 3.6051531430020987, + 3.6643769750007777, + 3.7326245340009336, + 3.807012820001546, + 3.8765420770032506, + 3.954172469002515, + 4.046650547999889, + 4.112342521002574, + 4.175299264999921, + 4.23776096200163, + 4.29212376400028, + 4.346141193000221, + 4.406239576001099, + 4.470648345002701, + 4.542288371001632, + 4.628536084001098, + 4.68735354599994, + 4.754679929999838, + 4.84613715100204, + 4.936513382002886, + 5.055991454002651, + 5.158364180002536, + 5.251543113001389, + 5.3191646209998, + 5.411908471000061, + 5.51450050800122, + 5.592487880003318, + 5.64557849600169, + 5.713603464002517, + 5.794314343002043, + 5.883868008000718, + 5.9584144210020895, + 6.028249641000002, + 6.120030022000719, + 6.192313685001864, + 6.262082999000995, + 6.337752391002141, + 6.392170825001813, + 6.452946718000021, + 6.548823455999809, + 6.655057697000302, + 6.750795780000772, + 6.836366704003012, + 6.9031485300001805, + 6.983332753003197, + 7.077792084000976, + 7.202592896002898, + 7.295453777001967, + 7.364519900002051, + 7.440002983003069, + 7.510080924999784, + 7.575407169002574, + 7.661527913001919, + 7.72789534900221, + 7.786154817000352, + 7.845734676000575, + 7.912444107001647, + 8.016722782002034 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.4680750022817, + "p95": 102.372725999885, + "max": 124.80081200192217 + } + }, + { + "start": 28858.254320548, + "first": 28858.434198063, + "last": 28866.272546948, + "end": 28866.272653097, + "elapsed_s": 8.018332548999751, + "chunk_offsets_s": [ + 0.17987751500186278, + 0.5014638770007878, + 0.5845109910005704, + 0.677380485998583, + 0.7479209180019097, + 0.8059203500015428, + 0.8630404620016634, + 0.9279779050011712, + 0.9924590390000958, + 1.0811542150004243, + 1.16242283199972, + 1.253887830000167, + 1.3388446499993734, + 1.4202024629994412, + 1.4863252530012687, + 1.552465504999418, + 1.635839598999155, + 1.7176741200019023, + 1.786556525999913, + 1.8699115540002822, + 1.9461426080015372, + 2.012694051998551, + 2.1082500029988296, + 2.210539924999466, + 2.2998943800012057, + 2.379058153001097, + 2.4478698189996067, + 2.5105409319985483, + 2.5656859279988566, + 2.6211060989990074, + 2.681897796999692, + 2.756101408001996, + 2.8219741670000076, + 2.877954667001177, + 2.937267494999105, + 3.003755571000511, + 3.0674055509989557, + 3.1401481079992664, + 3.2131734319991665, + 3.281691325999418, + 3.3544299290006165, + 3.4156248540020897, + 3.4761128470017866, + 3.5486488669994287, + 3.6061715579999145, + 3.6653688170008536, + 3.733338588001061, + 3.8076729429994884, + 3.8778522220018203, + 3.9551637670010678, + 4.04791124500116, + 4.113277214000846, + 4.176280365998537, + 4.238500596999074, + 4.292715996998595, + 4.347036010000011, + 4.407267803999275, + 4.471638105002057, + 4.543053747001977, + 4.62954921599885, + 4.6884937819995685, + 4.755604586000118, + 4.847259090001899, + 4.937478362000547, + 5.056439455001964, + 5.159613799001818, + 5.252556677001849, + 5.320189423000556, + 5.4131476690017735, + 5.515815999999177, + 5.593387483000697, + 5.646524443000089, + 5.7145393889986735, + 5.794922520999535, + 5.885020443001849, + 5.959527491999324, + 6.029601791000459, + 6.1212214039987884, + 6.193746242999623, + 6.263264680001157, + 6.339577626000391, + 6.393311141000595, + 6.453865899999073, + 6.550230786000611, + 6.656430977000127, + 6.75216544399882, + 6.83749534999879, + 6.904168146000302, + 6.984285695998551, + 7.079323011999804, + 7.203609741998662, + 7.296486504001223, + 7.365955596000276, + 7.440872171999217, + 7.511204161000933, + 7.576781106999988, + 7.662226743999781, + 7.729085834002035, + 7.786983025001973, + 7.847787288999825, + 7.912864765999984, + 8.018226399999548 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 71.4156419999199, + "p95": 103.17434399985359, + "max": 321.586361998925 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 161.42050330553056, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.991140422000171, + "end_to_end_tps": 160.17738800785804, + "timing_valid": true, + "steady_tps": 169.25184954965124, + "per_stream_tps": [ + 34.1, + 32.7, + 33.3, + 32.3, + 33.7 + ], + "ttft_s": [ + 0.51, + 0.18, + 0.34, + 0.09, + 0.42 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.5074329390008643, + "decode_tps": 34.08274609552388, + "prefill_tps": null, + "itl_mean_ms": 29.34035881960025, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.10267399996519, + "round_gap_p50_ms": 71.2792519989307, + "round_gap_p95_ms": 98.1511549980496, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1837756659988372, + "decode_tps": 32.66705499084132, + "prefill_tps": null, + "itl_mean_ms": 30.611880999997226, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.92043499974534, + "round_gap_p50_ms": 71.50863499919069, + "round_gap_p95_ms": 99.91603899834445, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33932728500076337, + "decode_tps": 33.329103687097906, + "prefill_tps": null, + "itl_mean_ms": 30.003807164700678, + "itl_p50_ms": 0.0, + "itl_p95_ms": 94.19452500151237, + "round_gap_p50_ms": 71.51183699897956, + "round_gap_p95_ms": 99.556355002278, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.09202823200030252, + "decode_tps": 32.287043630631224, + "prefill_tps": null, + "itl_mean_ms": 30.972176066664844, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.9236060003168, + "round_gap_p50_ms": 71.50917900071363, + "round_gap_p95_ms": 99.08355900188326, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.42415552500096965, + "decode_tps": 33.70351993611674, + "prefill_tps": null, + "itl_mean_ms": 29.670491447049084, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.9988110003469, + "round_gap_p50_ms": 71.29774500208441, + "round_gap_p95_ms": 99.49022599903401, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 30.119742899602414, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.92043499974534, + "round_gap_p50_ms": 71.50863499919069, + "round_gap_p95_ms": 99.556355002278, + "round_gap_mean_ms": 76.04489543959026, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28866.27662616, + "first": 28866.784059099, + "last": 28874.265850598, + "end": 28874.266588136, + "elapsed_s": 7.989961975999904, + "chunk_offsets_s": [ + 0.5074329390008643, + 0.578712190999795, + 0.6622408599978371, + 0.7579258599980676, + 0.8282052499998827, + 0.8845516269975633, + 0.9458096969974576, + 1.0249299009992683, + 1.0979603019986826, + 1.1850769439988653, + 1.2614251029990555, + 1.3559548529992753, + 1.4410606479978014, + 1.5201748959989345, + 1.5859589239989873, + 1.6626718679981423, + 1.7557745419981075, + 1.8382345409991103, + 1.9054882659984287, + 1.9884028439992107, + 2.070270209998853, + 2.1462308560003294, + 2.250915268999961, + 2.350208117000875, + 2.439550331000646, + 2.5175307569988945, + 2.5845339829975273, + 2.64713961400048, + 2.7048692509997636, + 2.7586570259991277, + 2.8184646800000337, + 2.892352505998133, + 2.955858860997978, + 3.0132857219978177, + 3.0780285359978734, + 3.1476046499992663, + 3.2089721739976085, + 3.276733699000033, + 3.3491847580007743, + 3.4164657069995883, + 3.4879776399975526, + 3.5451988280001387, + 3.6071645819974947, + 3.6802994979989307, + 3.7362028719981026, + 3.7967483949978487, + 3.8626530500005174, + 3.9236820949990943, + 3.991934133999166, + 4.073863870999048, + 4.16998028099988, + 4.228544026998861, + 4.289815434000047, + 4.3540625069981616, + 4.410735401997954, + 4.4641360030000214, + 4.523995517000003, + 4.590731304000656, + 4.65137320699796, + 4.72702089700033, + 4.786188445999869, + 4.8537942839975585, + 4.9451805429998785, + 5.034128956998757, + 5.144034953998926, + 5.239721046000341, + 5.331828353999299, + 5.401875494997512, + 5.489415858999564, + 5.578496222999092, + 5.654712384999584, + 5.707453815997724, + 5.777346315000614, + 5.856054748997849, + 5.934951947998343, + 6.001372308997816, + 6.073993325997435, + 6.155734989999473, + 6.220103602998279, + 6.287665207997634, + 6.362617615999625, + 6.417192333999992, + 6.477423083000758, + 6.563174206999975, + 6.661325361998024, + 6.757778994000546, + 6.8427795649986365, + 6.910450388000754, + 6.989889950000361, + 7.087657665000734, + 7.197585706999234, + 7.283343680999678, + 7.352994311000657, + 7.427929954999854, + 7.497729730999708, + 7.558812149000005, + 7.631882439000037, + 7.696628609999607, + 7.75476859400078, + 7.815913329999603, + 7.882631799999217, + 7.989224437998928 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 71.2792519989307, + "p95": 98.1511549980496, + "max": 109.9280419985007 + } + }, + { + "start": 28866.275628797, + "first": 28866.459404463, + "last": 28874.265434118, + "end": 28874.265661512, + "elapsed_s": 7.990032714998961, + "chunk_offsets_s": [ + 0.1837756659988372, + 0.5795657220005523, + 0.6631060289982997, + 0.7584923829999752, + 0.8293540169979678, + 0.8857859409981756, + 0.9467395539977588, + 1.0257747399991786, + 1.0991520819989091, + 1.186355760000879, + 1.262768127999152, + 1.3566885629988974, + 1.4421889559998817, + 1.5212175460001163, + 1.587018285998056, + 1.6637364640009764, + 1.7564093719993252, + 1.83878590400127, + 1.9065518700008397, + 1.989033510999434, + 2.0710000149992993, + 2.1484078180001234, + 2.251612338000996, + 2.3515283769993403, + 2.440669419000187, + 2.5184103489991685, + 2.5857893629981845, + 2.648039183997753, + 2.7057529890007572, + 2.7599501209988375, + 2.8198762020001595, + 2.89382743600072, + 2.956977707999613, + 3.0143249619977723, + 3.079523619999236, + 3.1483616849982354, + 3.2098938989984163, + 3.2779886629978137, + 3.35068418100127, + 3.4176624179999635, + 3.489171052999154, + 3.546476363000693, + 3.6084804239981167, + 3.681397502998152, + 3.7374421319982503, + 3.797991561001254, + 3.8638941509998403, + 3.925033186998917, + 3.99283812199792, + 4.075317301998439, + 4.171290968999529, + 4.22972250499879, + 4.290726930001256, + 4.3551472409999405, + 4.41085605400076, + 4.464835071998095, + 4.524881479999749, + 4.591979769000318, + 4.652584340998146, + 4.728135310000653, + 4.787485607001145, + 4.855086419000145, + 4.946280340998783, + 5.035197058998165, + 5.144956662999903, + 5.240780887997971, + 5.332416007000575, + 5.402429018999101, + 5.489875913001015, + 5.5791886179977155, + 5.655104292000033, + 5.708143633000873, + 5.777717253000446, + 5.856743364998692, + 5.936145216997829, + 6.0024684889976925, + 6.075205885001196, + 6.156633759997931, + 6.220760011998209, + 6.2887671509997745, + 6.363672928000597, + 6.417966097000317, + 6.477895912001259, + 6.563498512998194, + 6.662031458999991, + 6.758092750998912, + 6.843904736000695, + 6.911070425998332, + 6.990848622001067, + 7.089050105998467, + 7.198283223999169, + 7.283871496998472, + 7.353897547000088, + 7.4286578229985025, + 7.498635880998336, + 7.559718042000895, + 7.63313748300061, + 7.697864731999289, + 7.7555206659999385, + 7.817386307000561, + 7.884353269000712, + 7.98980532099813 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 71.50863499919069, + "p95": 99.91603899834445, + "max": 395.79005600171513 + } + }, + { + "start": 28866.275892465, + "first": 28866.61521975, + "last": 28874.266190577, + "end": 28874.266278589, + "elapsed_s": 7.990386123998178, + "chunk_offsets_s": [ + 0.33932728500076337, + 0.5791140719993564, + 0.6627154009984224, + 0.7583814789977623, + 0.8286489279998932, + 0.8850850620001438, + 0.9463946930009115, + 1.0258632000004582, + 1.0989443139987998, + 1.186045460999594, + 1.262299156998779, + 1.3564936820002913, + 1.4419853979998152, + 1.5208403169999656, + 1.5865039649979735, + 1.6633065530004387, + 1.7562511479991372, + 1.8384678090005764, + 1.9058060719980858, + 1.9884315300005255, + 2.070815009999933, + 2.1472124680003617, + 2.2514667289979116, + 2.3510230840001896, + 2.440060889999586, + 2.5182212149993575, + 2.585215684001014, + 2.6476969169998483, + 2.7053094069997314, + 2.759296208998421, + 2.8187822139989294, + 2.8932313619989145, + 2.956649304000166, + 3.0139561369978765, + 3.0789789150003344, + 3.148207703998196, + 3.2094938419977552, + 3.2775943060005375, + 3.3503603549979744, + 3.417298540000047, + 3.4888103769990266, + 3.546120201001031, + 3.6080336409977463, + 3.6811881350004114, + 3.737017407998792, + 3.7976209919979738, + 3.8637939889995323, + 3.924526197999512, + 3.9921917829997255, + 4.074515124997561, + 4.170768002997647, + 4.229678242998489, + 4.29086576699774, + 4.355351295998844, + 4.41084488100023, + 4.46481964099803, + 4.524842580998666, + 4.591552354999294, + 4.651936096997815, + 4.728048354998464, + 4.787026833000709, + 4.854586823999853, + 4.945848524999747, + 5.03503361699768, + 5.144966939998994, + 5.240366537000227, + 5.332675672998448, + 5.402328216998285, + 5.4900662649997685, + 5.579145299998345, + 5.655123325999739, + 5.708104141998774, + 5.777702749001037, + 5.856592600997828, + 5.935764114998165, + 6.002410700999462, + 6.074766769001144, + 6.156610803998774, + 6.22041116499895, + 6.288366741999198, + 6.363269557998137, + 6.4182472770007735, + 6.47785190599825, + 6.563944592999178, + 6.6621338939985435, + 6.758156766998582, + 6.843378088000463, + 6.911053728999832, + 6.990399964997778, + 7.08896747199833, + 7.1982249389984645, + 7.283866453999508, + 7.353690898999048, + 7.428701893000834, + 7.498604776999855, + 7.559613622001052, + 7.632943417000206, + 7.6975505749978765, + 7.755557243999647, + 7.816989756000112, + 7.883413630999712, + 7.9902981119994365 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 71.51183699897956, + "p95": 99.556355002278, + "max": 239.78678699859302 + } + }, + { + "start": 28866.275537487, + "first": 28866.367565719, + "last": 28874.265470616, + "end": 28874.266016297, + "elapsed_s": 7.990478809999331, + "chunk_offsets_s": [ + 0.09202823200030252, + 0.5795430229991325, + 0.6635098149999976, + 0.7589497640001355, + 0.8295634019996214, + 0.8856952720016125, + 0.946946634001506, + 1.0259446970012505, + 1.0988807140020072, + 1.1862271040008636, + 1.2624495519994525, + 1.3563731579997693, + 1.4421040180022828, + 1.5211281090014381, + 1.5869977319998725, + 1.6635379330000433, + 1.7566967960010516, + 1.8390364139995654, + 1.9063496250018943, + 1.9893658540022443, + 2.0714046499997494, + 2.1481321290011692, + 2.25213842499943, + 2.351221984001313, + 2.440212663001148, + 2.5181378289998975, + 2.5854597910001758, + 2.6479636910007684, + 2.7053036610013805, + 2.7594683280003665, + 2.819767844001035, + 2.8935477119994175, + 2.9568366300009075, + 3.0140481189991988, + 3.079271462000179, + 3.148616389000381, + 3.2093604959991353, + 3.2780160370020894, + 3.3505511219991604, + 3.4174830010015285, + 3.488992180002242, + 3.5462466159988253, + 3.6081060450014775, + 3.68117636900206, + 3.7371309049995034, + 3.797807868999371, + 3.8636062949990446, + 3.924400390998926, + 3.9922307779997936, + 4.07479571300064, + 4.1708677650021855, + 4.229558455001097, + 4.2910945219991845, + 4.35557835399959, + 4.411442412001634, + 4.465124402002402, + 4.525313217000075, + 4.591767631001858, + 4.652196339000511, + 4.728356653002265, + 4.787320899999031, + 4.854761200000212, + 4.946152870001242, + 5.03545765400122, + 5.144936941000196, + 5.2403681540017715, + 5.3326503970019985, + 5.402741271998821, + 5.490129801000876, + 5.579353707998962, + 5.655749048000871, + 5.708578107001813, + 5.778227107999555, + 5.857049615002325, + 5.9360672110015, + 6.002164561999962, + 6.075228809000691, + 6.156855679000728, + 6.220624887999293, + 6.28858427600062, + 6.363801969000633, + 6.418227588001173, + 6.478290798000671, + 6.563761653000256, + 6.662313761000405, + 6.758312974001456, + 6.843820374000643, + 6.911290711999754, + 6.990845549000369, + 7.08889713300232, + 7.1987111029993684, + 7.284262461002072, + 7.353803196001536, + 7.4290968909990625, + 7.4989970220012765, + 7.560074684999563, + 7.632751178000035, + 7.697589137998875, + 7.75548042200171, + 7.81753923100041, + 7.883960865001427, + 7.989933128999837 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 71.50917900071363, + "p95": 99.08355900188326, + "max": 487.51479099883 + } + }, + { + "start": 28866.275447714, + "first": 28866.699603239, + "last": 28874.265578558, + "end": 28874.265885464, + "elapsed_s": 7.990437749998819, + "chunk_offsets_s": [ + 0.42415552500096965, + 0.57983135899849, + 0.663483209998958, + 0.7587621179991402, + 0.829182187000697, + 0.885671130999981, + 0.9470885769987945, + 1.026168536998739, + 1.0990463809976063, + 1.1863829260000784, + 1.262660809999943, + 1.35665962100029, + 1.4422901590005495, + 1.5211285250006767, + 1.587034774998756, + 1.6638235479986179, + 1.7569002579984954, + 1.8396358030004194, + 1.9069913540006382, + 1.9889816459981375, + 2.071351918999426, + 2.148466520000511, + 2.252049228998658, + 2.351539454997692, + 2.4406356750005216, + 2.5187538339996536, + 2.5857528329979687, + 2.6482741979998536, + 2.705576789998304, + 2.7599189860011393, + 2.8193523720001394, + 2.893964752998727, + 2.9569695290010714, + 3.01421471499998, + 3.079650515999674, + 3.1484073879983043, + 3.2093358589991112, + 3.2783302429998002, + 3.3504379939986393, + 3.4181288439976925, + 3.489426588999777, + 3.546471528999973, + 3.608429358999274, + 3.681406531999528, + 3.7372538140007237, + 3.797962586999347, + 3.8644713209978363, + 3.9250867029986694, + 3.992773033998674, + 4.075684110001021, + 4.171706825000001, + 4.230234250000649, + 4.291065276000154, + 4.3554114699982165, + 4.411717430000863, + 4.46514285900048, + 4.525591467998311, + 4.592374927000492, + 4.652676083998813, + 4.7285809259992675, + 4.787528524000663, + 4.85489337800027, + 4.946198027999344, + 5.035229204000643, + 5.14532031699855, + 5.241058162999252, + 5.332806619000621, + 5.402711000999261, + 5.490127879998909, + 5.579808975999185, + 5.6554319450006005, + 5.708489967000787, + 5.778026367999701, + 5.857513366998319, + 5.9358476609995705, + 6.002683572998649, + 6.075119249999261, + 6.156684569999925, + 6.221117134999076, + 6.288283132998913, + 6.363628377999703, + 6.4185960440008785, + 6.478150503000506, + 6.563958007998735, + 6.662262455000018, + 6.758760990000155, + 6.844038947998342, + 6.911453066000831, + 6.991330543998629, + 7.089091310997901, + 7.198530916000891, + 7.284188728001027, + 7.354225790997589, + 7.428948721000779, + 7.498989690000599, + 7.560124902000098, + 7.633448182998109, + 7.697950230998686, + 7.756053492998035, + 7.817206937001174, + 7.884265738000977, + 7.990130843998486 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 71.29774500208441, + "p95": 99.49022599903401, + "max": 155.67583399752039 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 158.72427244570144, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 8.125957879001362, + "end_to_end_tps": 157.5198910774203, + "timing_valid": true, + "steady_tps": 164.8733904082225, + "per_stream_tps": [ + 33.2, + 32.1, + 31.7, + 32.5, + 32.8 + ], + "ttft_s": [ + 0.44, + 0.19, + 0.09, + 0.27, + 0.35 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.4443403330005822, + "decode_tps": 33.20200996150198, + "prefill_tps": null, + "itl_mean_ms": 30.118658513731813, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.44614799920237, + "round_gap_p50_ms": 74.10127199909766, + "round_gap_p95_ms": 105.46009300014703, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1862666499982879, + "decode_tps": 32.123025922614765, + "prefill_tps": null, + "itl_mean_ms": 31.130317623533568, + "itl_p50_ms": 0.0, + "itl_p95_ms": 94.03586700136657, + "round_gap_p50_ms": 74.07699099712772, + "round_gap_p95_ms": 105.73174000091967, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.0917686409993621, + "decode_tps": 31.745323014154117, + "prefill_tps": null, + "itl_mean_ms": 31.50070325490578, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.84902199963108, + "round_gap_p50_ms": 74.02398799968068, + "round_gap_p95_ms": 105.34245599774295, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2708184479997726, + "decode_tps": 32.469722234859574, + "prefill_tps": null, + "itl_mean_ms": 30.79792283921658, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.66492999834009, + "round_gap_p50_ms": 73.91963199916063, + "round_gap_p95_ms": 106.32058799819788, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.35489965799934, + "decode_tps": 32.81876630906352, + "prefill_tps": null, + "itl_mean_ms": 30.470371450977765, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.90930800145725, + "round_gap_p50_ms": 74.01466800001799, + "round_gap_p95_ms": 106.10233400075231, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 30.8035947364731, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.7647079990711, + "round_gap_p50_ms": 74.02398799968068, + "round_gap_p95_ms": 106.04054400027962, + "round_gap_mean_ms": 77.77145205743209, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 28874.268901294, + "first": 28874.713241627, + "last": 28882.393499548, + "end": 28882.394661105, + "elapsed_s": 8.125759811002354, + "chunk_offsets_s": [ + 0.4443403330005822, + 0.5267199070003699, + 0.6109724620000634, + 0.7039563470025314, + 0.7755156210005225, + 0.8321930940001039, + 0.8895076379994862, + 0.956982355000946, + 1.0334421930019744, + 1.1305664230021648, + 1.2104955020004127, + 1.3036271260025387, + 1.3907915220006544, + 1.4738462140012416, + 1.5523780319999787, + 1.6255143630005477, + 1.7080179270014924, + 1.791453141002421, + 1.8573626079996757, + 1.939450283000042, + 2.0167072050026036, + 2.088088914002583, + 2.19413005000024, + 2.299590143000387, + 2.388315877000423, + 2.465877910999552, + 2.5343076470016968, + 2.5975377850008954, + 2.6539093560022593, + 2.7072557320025226, + 2.774639790000947, + 2.860530308000307, + 2.9275955660014006, + 2.994938980002189, + 3.0602285330023733, + 3.1311460390024877, + 3.193665934002638, + 3.259579608002241, + 3.3338268760016945, + 3.4141373700003896, + 3.490765990001819, + 3.549487231000967, + 3.6118463710008655, + 3.685947642999963, + 3.742201360000763, + 3.8099879840010544, + 3.885385617002612, + 3.945811004999996, + 4.013120253999659, + 4.097258068002702, + 4.203190516000177, + 4.269728418999875, + 4.3304857820003235, + 4.391680324002664, + 4.44525695700213, + 4.499659428001905, + 4.559825549000379, + 4.624771693001094, + 4.689030565001303, + 4.777672085001541, + 4.840831767000054, + 4.906278353999369, + 4.998894807002216, + 5.092659515001287, + 5.211183731000347, + 5.313501886001177, + 5.406948034000379, + 5.473910803000763, + 5.563689451002574, + 5.6535372830003325, + 5.733295155001542, + 5.7988404290008475, + 5.872072540001682, + 5.951283450001938, + 6.029939538999315, + 6.104399577001459, + 6.175409904000844, + 6.2524240519996965, + 6.322377963002509, + 6.402661473999615, + 6.477756612999656, + 6.533736444001988, + 6.594802049999998, + 6.679694231999747, + 6.776488817002246, + 6.873586417001206, + 6.958320183999604, + 7.02484032600114, + 7.110720699001831, + 7.205328940999607, + 7.312473120000504, + 7.399143807000655, + 7.467401619000157, + 7.541701301001012, + 7.612789119000809, + 7.674668137999106, + 7.745991273000982, + 7.811535101001937, + 7.869535764002649, + 7.930505208001705, + 7.996485046001908, + 8.124598254002194 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 74.10127199909766, + "p95": 105.46009300014703, + "max": 128.11320800028625 + } + }, + { + "start": 28874.26879809, + "first": 28874.45506474, + "last": 28882.393295734, + "end": 28882.394019056, + "elapsed_s": 8.125220965997869, + "chunk_offsets_s": [ + 0.1862666499982879, + 0.5267258620006032, + 0.6116497449984308, + 0.7038332140000421, + 0.7754032609991555, + 0.831927937000728, + 0.8894916940007533, + 0.9569119679981668, + 1.0334787079991656, + 1.13059059599982, + 1.210468368997681, + 1.3033204610001121, + 1.3906002910007373, + 1.473300965000817, + 1.5522224499982258, + 1.625895273999049, + 1.708352031000686, + 1.7917392039998958, + 1.8577755029982654, + 1.9398290009994525, + 2.0162411159981275, + 2.088092340000003, + 2.194643859998905, + 2.2992317879979964, + 2.387951406999491, + 2.4663989879991277, + 2.5348166059993673, + 2.5979772219980077, + 2.6543964480006252, + 2.7075045609999506, + 2.7755254809999315, + 2.8602317910008423, + 2.927754285999981, + 2.995098052000685, + 3.060606306000409, + 3.1314344389975304, + 3.193657402000099, + 3.2597545600001467, + 3.3338315509972745, + 3.4136628929991275, + 3.490970844999538, + 3.5493452080008865, + 3.6114337489998434, + 3.6856124039986753, + 3.7426782229995297, + 3.810160614997585, + 3.8855674999977055, + 3.945724977998907, + 4.0133279750007205, + 4.097640755997418, + 4.203372495998337, + 4.2701250499994785, + 4.3307684200008225, + 4.391986172999168, + 4.445553858000494, + 4.500046405999456, + 4.5603169789974345, + 4.625286078997306, + 4.689084801000718, + 4.7783982410001045, + 4.840464895998593, + 4.906189477998851, + 4.9989448960004665, + 5.092808180997963, + 5.211477749999176, + 5.31354086599822, + 5.407576732999587, + 5.474255439999979, + 5.563858335997793, + 5.6537501419988985, + 5.733684533999622, + 5.799060219000239, + 5.872502562997397, + 5.951248184999713, + 6.02982540299854, + 6.104289857998083, + 6.175197607000882, + 6.2524157439984265, + 6.322739391998766, + 6.403042912999808, + 6.479127971997514, + 6.533982839999226, + 6.59532732200023, + 6.680106806998083, + 6.77696671999729, + 6.874118683997949, + 6.958217187999253, + 7.024872134999896, + 7.110525033000158, + 7.205821971998375, + 7.311986605000129, + 7.399753825000516, + 7.467395600997406, + 7.542070381998201, + 7.6130192979981075, + 7.674083076999523, + 7.745798377000028, + 7.811330790998909, + 7.869342002999474, + 7.930123897000158, + 7.9970267089993285, + 8.124497643999348 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 74.07699099712772, + "p95": 105.73174000091967, + "max": 340.4592120023153 + } + }, + { + "start": 28874.268981078, + "first": 28874.360749719, + "last": 28882.393429049, + "end": 28882.394264267, + "elapsed_s": 8.125283189001493, + "chunk_offsets_s": [ + 0.0917686409993621, + 0.5264553900015017, + 0.610808123001334, + 0.7036017380014528, + 0.775119886999164, + 0.8316946680024557, + 0.8892627150016779, + 0.9568153590007569, + 1.0330187170002318, + 1.1303515799991146, + 1.210229096999683, + 1.3030728160010767, + 1.3903626350002014, + 1.4729286190013227, + 1.5521441710006911, + 1.6253766139998334, + 1.708225359001517, + 1.791592010002205, + 1.8575548160006292, + 1.9396841439993295, + 2.0168559350022406, + 2.087715671001206, + 2.1937562150014855, + 2.298944059999485, + 2.3878592159999243, + 2.4660389040000155, + 2.534399452000798, + 2.597618722000334, + 2.654017937999015, + 2.707510115000332, + 2.775422869999602, + 2.860214967000502, + 2.9274273539995193, + 2.995204040002136, + 3.0603698840022844, + 3.130970846999844, + 3.193427831000008, + 3.2594116200016288, + 3.3334356080013094, + 3.413706995001121, + 3.490921012002218, + 3.5491102100022545, + 3.6112110290014243, + 3.68525214300098, + 3.742660069001431, + 3.809243162999337, + 3.885248236001644, + 3.945499920999282, + 4.013083548001305, + 4.097370669001975, + 4.202713124999718, + 4.2697560810011055, + 4.330656972000725, + 4.3917635649995646, + 4.445414268000604, + 4.4998221330024535, + 4.559921069001575, + 4.624930156001938, + 4.68866916800107, + 4.778060183001799, + 4.840231499001675, + 4.905945579001127, + 4.998860406001768, + 5.092172983000637, + 5.2109245770006964, + 5.313290116002463, + 5.407139138002094, + 5.47397997400185, + 5.563548964000802, + 5.6537904659999185, + 5.73327898700154, + 5.798339714001486, + 5.872073051999905, + 5.9506172130022605, + 6.0300414619996445, + 6.104213211001479, + 6.175377343999571, + 6.25217235799937, + 6.322127933999582, + 6.402250564999122, + 6.478007235000405, + 6.53358595300233, + 6.595014733000426, + 6.6796990990005725, + 6.776897117000772, + 6.873410440999578, + 6.958185700001195, + 7.024514787000953, + 7.110536575000879, + 7.205584797000483, + 7.31198407500051, + 7.39944728699993, + 7.467362976000004, + 7.541720110999449, + 7.612583400001313, + 7.673932858000626, + 7.745708378999552, + 7.811224545999721, + 7.869322280999768, + 7.930630022001424, + 7.997157077999873, + 8.124447971000336 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 74.02398799968068, + "p95": 105.34245599774295, + "max": 434.68674900213955 + } + }, + { + "start": 28874.269088219, + "first": 28874.539906667, + "last": 28882.393376991, + "end": 28882.3944174, + "elapsed_s": 8.12532918099896, + "chunk_offsets_s": [ + 0.2708184479997726, + 0.5268976600018505, + 0.6114154360002431, + 0.7038285560010991, + 0.775468574000115, + 0.8318529239986674, + 0.8895005310005217, + 0.9569632099992305, + 1.0333380459996988, + 1.1306413089987473, + 1.210445990000153, + 1.303097821000847, + 1.390365085000667, + 1.473466889001429, + 1.552280960000644, + 1.6253784559994529, + 1.7078999639998074, + 1.7912257319985656, + 1.8572090429988748, + 1.9393021770010819, + 2.0158165840002766, + 2.087685513000906, + 2.194006100999104, + 2.2990926310012583, + 2.3878562710015103, + 2.465742868000234, + 2.534024849999696, + 2.597300819001248, + 2.653762499001459, + 2.7069521410012385, + 2.7741227639999124, + 2.859444805999374, + 2.9275115730015386, + 2.9950380379996204, + 3.0601855360000627, + 3.13109894399895, + 3.1929987850016914, + 3.2595711239991942, + 3.333490755998355, + 3.4138478499990015, + 3.4908597499997995, + 3.549187818000064, + 3.6114290750010696, + 3.685613572000875, + 3.7424311239992676, + 3.8093691149988445, + 3.884939425999619, + 3.9456749700002547, + 4.013105206999171, + 4.096764334000909, + 4.203197351998824, + 4.2697706809995, + 4.330360616000689, + 4.391443582000647, + 4.445112116998644, + 4.4995332209982735, + 4.559581539000646, + 4.624702586999774, + 4.688504493999062, + 4.777664545999869, + 4.84046635300183, + 4.9060409080011596, + 4.998937197000487, + 5.092175721001695, + 5.2111302480007, + 5.313011097001436, + 5.406676026999776, + 5.473667482001474, + 5.563265766999393, + 5.653309985998931, + 5.733057020999695, + 5.7984191620016645, + 5.8717966900003375, + 5.950843151000299, + 6.029350493001402, + 6.103864333999809, + 6.174967779999861, + 6.251961597001355, + 6.322504059000494, + 6.402938268998696, + 6.478628281998681, + 6.533881893999933, + 6.595476435999444, + 6.680083483999624, + 6.776427682998474, + 6.873736107998411, + 6.958215332000691, + 7.024545699998271, + 7.110111466001399, + 7.205031689001771, + 7.312106600998959, + 7.399105805001454, + 7.467171456999495, + 7.541851793001115, + 7.612522889998218, + 7.674086646999058, + 7.7456652379987645, + 7.811167766001745, + 7.869734727999457, + 7.93007443299939, + 7.996585944998515, + 8.124288772 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 73.91963199916063, + "p95": 106.32058799819788, + "max": 256.07921200207784 + } + }, + { + "start": 28874.268703226, + "first": 28874.623602884, + "last": 28882.393547604, + "end": 28882.393760367, + "elapsed_s": 8.125057140998251, + "chunk_offsets_s": [ + 0.35489965799934, + 0.52741595200132, + 0.6116378359984083, + 0.7040970259986352, + 0.7756642239983194, + 0.8322900870007288, + 0.8898261499998625, + 0.9572732050000923, + 1.0334063039990724, + 1.1308169170006295, + 1.2106267210001533, + 1.3035569950006902, + 1.3909449279999535, + 1.4734649199999694, + 1.5526248440000927, + 1.6255855039999005, + 1.7083674790010264, + 1.7916970240003138, + 1.8577029070002027, + 1.9397718049985997, + 2.0167918400002236, + 2.0881286789990554, + 2.1942310129998077, + 2.299720545001037, + 2.388281860999996, + 2.4661963590006053, + 2.5345600139989983, + 2.597792330001539, + 2.654066906998196, + 2.707395738998457, + 2.7750851009986945, + 2.8599769760003255, + 2.9274976939996122, + 2.995080043001508, + 3.060480820000521, + 3.131412268998247, + 3.1933197299986205, + 3.260094217999722, + 3.334127571000863, + 3.4138330269997823, + 3.49115874799827, + 3.5497727190013393, + 3.6117578800003685, + 3.6857725480003865, + 3.74229503999959, + 3.8096969750004064, + 3.885170918001677, + 3.94552790400121, + 4.013192899001297, + 4.0970161559998814, + 4.203267963999679, + 4.2698409570002696, + 4.330794129000424, + 4.3919371890006005, + 4.445549232001213, + 4.49997915700078, + 4.560075867000705, + 4.6250364820007235, + 4.688802019998548, + 4.777808329999971, + 4.840674040999147, + 4.906521821001661, + 4.998907741999574, + 5.092636223998852, + 5.211453962998348, + 5.313446610998653, + 5.407355919000111, + 5.474160385001596, + 5.5637317770015216, + 5.654184119001002, + 5.733656057000189, + 5.798885347998294, + 5.872132811000483, + 5.951080438000645, + 6.0304833320005855, + 6.104554918998474, + 6.1755485350004165, + 6.252670191999641, + 6.322535450999567, + 6.40276146099859, + 6.478569244998653, + 6.534500316000049, + 6.595718461998331, + 6.680157489001431, + 6.777214524998271, + 6.874068771001475, + 6.958193016998848, + 7.024708404998819, + 7.111071418999927, + 7.205281558999559, + 7.311996770000405, + 7.399662452000484, + 7.467406102998211, + 7.542291837999073, + 7.612730594999448, + 7.674052630998631, + 7.746369863001746, + 7.811626668000827, + 7.869988967999234, + 7.930802861999837, + 7.997198556000512, + 8.12484437799867 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 74.01466800001799, + "p95": 106.10233400075231, + "max": 172.51629400198 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/receipts/flashnext-cuda-sampled-heads/A2.json b/docs/receipts/flashnext-cuda-sampled-heads/A2.json new file mode 100644 index 0000000000..771f9dcbdb --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/A2.json @@ -0,0 +1,23728 @@ +{ + "label": "A2", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "code", + "temperature": 1.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.7, + "per_stream_tps": 67.7, + "ttft_s_max": 0.11, + "request": { + "ttft_s": 0.097, + "decode_tps": 67.658, + "prefill_tps": 144.672, + "itl_mean_ms": 14.78, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.923, + "round_gap_p50_ms": 65.215, + "round_gap_p95_ms": 81.995, + "tokens_per_chunk": 4.414 + }, + "pooled": { + "aggregate_prefill_tps": 144.672, + "itl_mean_ms": 14.78, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.923, + "round_gap_p50_ms": 65.215, + "round_gap_p95_ms": 81.995, + "tokens_per_chunk": 4.414 + }, + "timing_valid": true, + "end_to_end_tps": 66.2431223570263, + "makespan_s": 3.8645521359976556, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.46751601224722, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8926193229999626, + "end_to_end_tps": 65.76548559151323, + "timing_valid": true, + "per_stream_tps": [ + 67.5 + ], + "ttft_s": [ + 0.11 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.11286016300073243, + "decode_tps": 67.46751601224722, + "prefill_tps": 124.04731331027003, + "itl_mean_ms": 14.821947792156339, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.03769199745147, + "round_gap_p50_ms": 65.21486899873707, + "round_gap_p95_ms": 81.9948860007571, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.821947792156339, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.03769199745147, + "round_gap_p50_ms": 65.21486899873707, + "round_gap_p95_ms": 81.9948860007571, + "round_gap_mean_ms": 66.3087138070152, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 124.04731331027003 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 29607.898158073, + "first": 29608.011018236, + "last": 29611.790614923, + "end": 29611.790777396, + "elapsed_s": 3.8926193229999626, + "chunk_offsets_s": [ + 0.11286016300073243, + 0.16700060299990582, + 0.21999641900038114, + 0.27870365399940056, + 0.34521537399996305, + 0.40695797000080347, + 0.4690055739993113, + 0.5358976349998557, + 0.6109182930013048, + 0.6844207409994851, + 0.7447701410019363, + 0.8014497139993182, + 0.8565212249995966, + 0.91324756999893, + 0.9863115580010344, + 1.0692197480020695, + 1.143008656999882, + 1.2250035430006392, + 1.3038751400017645, + 1.375103426998976, + 1.4525628090013925, + 1.5234558779993677, + 1.6019359640013136, + 1.679973655998765, + 1.7611686269992788, + 1.8299931020010263, + 1.883398321999266, + 1.9378558040007192, + 1.986607486000139, + 2.0356947510008467, + 2.092016827999032, + 2.1481151090010826, + 2.208577499000967, + 2.2852235630016366, + 2.3504384320003737, + 2.396189195998886, + 2.4442994769997313, + 2.4961144699991564, + 2.5457524199991894, + 2.6034404749989335, + 2.684897776998696, + 2.7646874130005017, + 2.828619879001053, + 2.8789886660015327, + 2.942013562002103, + 3.021837847001734, + 3.1091867069990258, + 3.188723155999469, + 3.2639373439997144, + 3.328731705001701, + 3.3873878349986626, + 3.4500910230017325, + 3.5093106730018917, + 3.588053964998835, + 3.6681462160013325, + 3.7431013220011664, + 3.8203727250001975, + 3.892456850000599 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 65.21486899873707, + "p95": 81.9948860007571, + "max": 87.34885999729158 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.72940216441444, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.861863614998583, + "end_to_end_tps": 66.28923895855756, + "timing_valid": true, + "per_stream_tps": [ + 67.7 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09677038899826584, + "decode_tps": 67.72940216441444, + "prefill_tps": 144.67235426997078, + "itl_mean_ms": 14.764636450982994, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.92325600166805, + "round_gap_p50_ms": 64.641536999261, + "round_gap_p95_ms": 82.24530100051197, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.764636450982994, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.92325600166805, + "round_gap_p50_ms": 64.641536999261, + "round_gap_p95_ms": 82.24530100051197, + "round_gap_mean_ms": 66.05232096492392, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 144.67235426997078 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 29611.791405656, + "first": 29611.888176045, + "last": 29615.65315834, + "end": 29615.653269271, + "elapsed_s": 3.861863614998583, + "chunk_offsets_s": [ + 0.09677038899826584, + 0.15775158199903672, + 0.2148883179979748, + 0.27546401799918385, + 0.34056661399881705, + 0.40262288200028706, + 0.4654402320011286, + 0.5342242840015388, + 0.6090539850010828, + 0.6828151900008379, + 0.7422088179991988, + 0.7982334150001407, + 0.8540581070010376, + 0.909262929999386, + 0.9822761979994539, + 1.0582611270001507, + 1.1291591529989091, + 1.2152588050012127, + 1.294639528998232, + 1.3648630210009287, + 1.4408810470013123, + 1.5106838760002574, + 1.5883993729985377, + 1.6664986590003537, + 1.7457617369982472, + 1.8141664809991198, + 1.867142914001306, + 1.921911795001506, + 1.9680023190012434, + 2.012277654001082, + 2.0664403080008924, + 2.122699890998774, + 2.186154160001024, + 2.265607329998602, + 2.330089062998013, + 2.3743278510009986, + 2.4212944870014326, + 2.472455598999659, + 2.5223937349983316, + 2.580363869001303, + 2.661327794998215, + 2.7405452259990852, + 2.803861323998717, + 2.853864541000803, + 2.9149246790002508, + 2.9922694669985503, + 3.0745147679990623, + 3.158397752999008, + 3.2376310709987592, + 3.3022726079980202, + 3.360825822001061, + 3.421788153998932, + 3.480664557999262, + 3.5597712179987866, + 3.6380789860013465, + 3.712513014997967, + 3.790436270999635, + 3.8617526839989296 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 64.641536999261, + "p95": 82.24530100051197, + "max": 86.09965200230363 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.65784936632899, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8645521359976556, + "end_to_end_tps": 66.2431223570263, + "timing_valid": true, + "per_stream_tps": [ + 67.7 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09551880099752452, + "decode_tps": 67.65784936632899, + "prefill_tps": 146.56800393006216, + "itl_mean_ms": 14.780251062749063, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.44816900230944, + "round_gap_p50_ms": 67.8777210014232, + "round_gap_p95_ms": 81.04573199670995, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.780251062749063, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.44816900230944, + "round_gap_p50_ms": 67.8777210014232, + "round_gap_p95_ms": 81.04573199670995, + "round_gap_mean_ms": 66.12217580703528, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 146.56800393006216 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 29615.653889769, + "first": 29615.74940857, + "last": 29619.518372591, + "end": 29619.518441905, + "elapsed_s": 3.8645521359976556, + "chunk_offsets_s": [ + 0.09551880099752452, + 0.149712845999602, + 0.20425616499778698, + 0.2657711770007154, + 0.3357874459979939, + 0.3986713099984627, + 0.4618836169975111, + 0.5297613379989343, + 0.6042131619979045, + 0.6790468889994372, + 0.7390751099992485, + 0.794994964999205, + 0.8498668699976406, + 0.9057009389980522, + 0.9778995150008996, + 1.0535117190011078, + 1.125629731999652, + 1.2067407869981253, + 1.2889482479986327, + 1.363085434997629, + 1.4403877289987577, + 1.508262511000794, + 1.586038010998891, + 1.6644825599978503, + 1.743559747999825, + 1.811579700999573, + 1.86486299399985, + 1.9196858020004584, + 1.965900048999174, + 2.0102893500006758, + 2.0632624459976796, + 2.1185954229986237, + 2.1778808630006097, + 2.2570009359988035, + 2.3270609469982446, + 2.373262586999772, + 2.420019441000477, + 2.4720939439976064, + 2.5214827979980328, + 2.57954407500074, + 2.6598373640008504, + 2.7390058579985634, + 2.802354669998749, + 2.852949014999467, + 2.916325540998514, + 2.9937737100008235, + 3.0748194419975334, + 3.155065154998738, + 3.230618574998516, + 3.2991875619991333, + 3.3610479089984437, + 3.4233642809995217, + 3.4824304959984147, + 3.561241720999533, + 3.6410838480005623, + 3.7168924899997364, + 3.793988352997985, + 3.8644828219985357 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 67.8777210014232, + "p95": 81.04573199670995, + "max": 82.20746100050746 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 77.9, + "per_stream_tps": 44.5, + "ttft_s_max": 0.2, + "request": { + "ttft_s": 0.14, + "decode_tps": 44.422, + "prefill_tps": 111.433, + "itl_mean_ms": 22.783, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.676, + "round_gap_p50_ms": 85.376, + "round_gap_p95_ms": 111.128, + "tokens_per_chunk": 3.73 + }, + "pooled": { + "aggregate_prefill_tps": 148.846, + "itl_mean_ms": 22.793, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.512, + "round_gap_p50_ms": 85.58, + "round_gap_p95_ms": 112.264, + "tokens_per_chunk": 3.631 + }, + "timing_valid": true, + "end_to_end_tps": 77.14309220929216, + "makespan_s": 6.637016813001537, + "steady_tps": 85.5, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 77.93656702650048, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.637016813001537, + "end_to_end_tps": 77.14309220929216, + "timing_valid": true, + "steady_tps": 85.54998126064977, + "per_stream_tps": [ + 50.2, + 39.0 + ], + "ttft_s": [ + 0.19, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18509735300176544, + "decode_tps": 50.19024827161623, + "prefill_tps": 75.63587362519695, + "itl_mean_ms": 19.92418914902088, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.14143100063666, + "round_gap_p50_ms": 88.43987399814068, + "round_gap_p95_ms": 111.54153100142139, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09307973199975095, + "decode_tps": 38.96828351325024, + "prefill_tps": 150.40868403056274, + "itl_mean_ms": 25.661895003919625, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.0051380029181, + "round_gap_p50_ms": 82.42536699981429, + "round_gap_p95_ms": 108.10357800073689, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 22.793042076470254, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.5115729997924, + "round_gap_p50_ms": 85.32210700286669, + "round_gap_p95_ms": 111.2185829988448, + "round_gap_mean_ms": 83.62914718704913, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 151.16585450468287 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 29619.519865369, + "first": 29619.704962722, + "last": 29624.785630955, + "end": 29624.785846435, + "elapsed_s": 5.265981066000677, + "chunk_offsets_s": [ + 0.18509735300176544, + 0.2521604460016533, + 0.314960488001816, + 0.38424887899964233, + 0.46832128599999123, + 0.5432442910023383, + 0.6187836390017765, + 0.7048153690011532, + 0.7974398550031765, + 0.9002391130015894, + 0.9854605230029847, + 1.0564106020028703, + 1.1394197900008294, + 1.211518760999752, + 1.2968408680026187, + 1.3852807420007593, + 1.482422173001396, + 1.5746601110004121, + 1.6623149970000668, + 1.744155751002836, + 1.8362934780016076, + 1.9227853910015256, + 2.0177653930004453, + 2.106792727001448, + 2.2045672460008063, + 2.2979725700024574, + 2.371391434000543, + 2.45461734300261, + 2.5286652040012996, + 2.5994074100017315, + 2.675929452001583, + 2.7490281200007303, + 2.833800231001078, + 2.9392953570022655, + 3.029572575000202, + 3.093265159001021, + 3.153398605001712, + 3.2283206550018804, + 3.318971104999946, + 3.417318118001276, + 3.5068730910024897, + 3.6025919300009264, + 3.7042016699997475, + 3.8009686110017356, + 3.9091933490017254, + 4.022848511001939, + 4.127124852002453, + 4.2189644320023945, + 4.313316076000774, + 4.407586082001217, + 4.4750385690022085, + 4.5587301360028505, + 4.639825278001808, + 4.731900137001503, + 4.8399864610000805, + 4.951527992001502, + 5.070126896000147, + 5.170891260000644, + 5.2657655860020895 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 88.73360399957164, + "p95": 111.54153100142139, + "max": 118.59890399864526 + } + }, + { + "start": 29619.519735707, + "first": 29619.612815439, + "last": 29626.156598665, + "end": 29626.15675252, + "elapsed_s": 6.637016813001537, + "chunk_offsets_s": [ + 0.09307973199975095, + 0.2525506400015729, + 0.31539850500121247, + 0.3843097270000726, + 0.46848890999899595, + 0.5431463089989848, + 0.6190209359992878, + 0.7051721499992709, + 0.7973471850018541, + 0.8999057260007248, + 0.9858144099998754, + 1.0566350020017126, + 1.1390603690015269, + 1.2115836160010076, + 1.2976692430020194, + 1.3851497169998765, + 1.482661289999669, + 1.5743216629998642, + 1.662511763999646, + 1.7443445160024567, + 1.8361841530022502, + 1.922859578000498, + 2.018139722000342, + 2.10696997299965, + 2.204975111002568, + 2.2980630509991897, + 2.371401500000502, + 2.4546794970010524, + 2.5288393789996917, + 2.599609545999556, + 2.676143538999895, + 2.749126796999917, + 2.8339586980000604, + 2.9394587259994296, + 3.029666159000044, + 3.0933237960016413, + 3.1533197579992702, + 3.2284998100003577, + 3.3192427339999995, + 3.4173823519995494, + 3.5070638350007357, + 3.6027814480003144, + 3.704388561000087, + 3.8011328169995977, + 3.9092363950003346, + 4.023036023001623, + 4.127031726002315, + 4.218769375002012, + 4.313497299001028, + 4.407367923002312, + 4.4752789810008835, + 4.559198209000897, + 4.640009718001238, + 4.732399627002451, + 4.840230057001463, + 4.951448640000308, + 5.0698520639998605, + 5.170583130999148, + 5.266647866999847, + 5.324424722999538, + 5.372602956002083, + 5.422485514001892, + 5.47899632500048, + 5.541112718001386, + 5.599228671002493, + 5.658222718000616, + 5.737116129999777, + 5.797380943000462, + 5.842147991999809, + 5.8879491170009715, + 5.935107338002126, + 5.995560308001586, + 6.073133608999342, + 6.150413947001653, + 6.22311287300181, + 6.296498416999384, + 6.373427703001653, + 6.443752003000554, + 6.492525259000104, + 6.539399453002261, + 6.5851727170011145, + 6.636862957999256 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 82.42536699981429, + "p95": 108.10357800073689, + "max": 159.47090800182195 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 78.01921285532701, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.632132948998333, + "end_to_end_tps": 77.19989993224256, + "timing_valid": true, + "steady_tps": 85.5076019689965, + "per_stream_tps": [ + 49.3, + 39.6 + ], + "ttft_s": [ + 0.1, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09508874699895387, + "decode_tps": 49.2723243348788, + "prefill_tps": 147.23088106475967, + "itl_mean_ms": 20.295368921577783, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.61201800210983, + "round_gap_p50_ms": 89.01821800100151, + "round_gap_p95_ms": 113.73574000026565, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.1881136019983387, + "decode_tps": 39.5721632264065, + "prefill_tps": 74.42311375295253, + "itl_mean_ms": 25.270288972544723, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.73935699922731, + "round_gap_p50_ms": 79.76460500140092, + "round_gap_p95_ms": 110.71412199817132, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 22.78282894706125, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.61201800210983, + "round_gap_p50_ms": 85.74967499953345, + "round_gap_p95_ms": 113.19698199804407, + "round_gap_mean_ms": 83.59167455396575, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 148.84622750590506 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 29626.158270599, + "first": 29626.253359346, + "last": 29631.428678421, + "end": 29631.428840426, + "elapsed_s": 5.270569827000145, + "chunk_offsets_s": [ + 0.09508874699895387, + 0.2528309769986663, + 0.3154449799985741, + 0.38350615899980767, + 0.46335415199791896, + 0.5365834420008468, + 0.6146266280011332, + 0.7025713609982631, + 0.7985495440007071, + 0.9051171969986171, + 0.9913708229978511, + 1.0618712949981273, + 1.1435417089996918, + 1.2159451779989467, + 1.3016948529984802, + 1.390320784998039, + 1.4822614750009961, + 1.5718514709988085, + 1.66086968899981, + 1.744688981998479, + 1.8423010000005888, + 1.9312813889991958, + 2.02690850900035, + 2.116081715001201, + 2.2136266929992416, + 2.307813781000732, + 2.379010417000245, + 2.455578068998875, + 2.528222753000591, + 2.5986029009982303, + 2.676163575000828, + 2.750463630000013, + 2.8420017059979727, + 2.948131689998263, + 3.036248368000088, + 3.101854889999231, + 3.1615848779983935, + 3.23692661899986, + 3.3285124100002577, + 3.4206483630005096, + 3.5056035070010694, + 3.601367871000548, + 3.7032528199997614, + 3.8055500849986856, + 3.9156915029998345, + 4.028350625998428, + 4.135013520000939, + 4.225922221001383, + 4.320280186999298, + 4.409355171999778, + 4.472402860999864, + 4.5525836249980784, + 4.634047071998793, + 4.726677463000669, + 4.840413203000935, + 4.953610184998979, + 5.072438514998794, + 5.175373684000078, + 5.270407822001289 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 89.04660150074051, + "p95": 113.73574000026565, + "max": 157.74222999971244 + } + }, + { + "start": 29626.158173446, + "first": 29626.346287048, + "last": 29632.790210736, + "end": 29632.790306395, + "elapsed_s": 6.632132948998333, + "chunk_offsets_s": [ + 0.1881136019983387, + 0.25266823199854116, + 0.3159822380002879, + 0.38394086299740593, + 0.46370546799880685, + 0.5369800529988424, + 0.6150994729978265, + 0.7030957030001446, + 0.7983920599981502, + 0.9056557369985967, + 0.9917479189971345, + 1.062280803998874, + 1.14419226300015, + 1.2163265290000709, + 1.3020248909997463, + 1.390630934998626, + 1.4820336829980079, + 1.5723458849970484, + 1.6611722029992961, + 1.7448954459978268, + 1.842634802997054, + 1.9315331849975337, + 2.0274104709969833, + 2.11605514799885, + 2.2140628099987225, + 2.308217142999638, + 2.37957653400008, + 2.4559196819973295, + 2.5285185409993574, + 2.5989802659969428, + 2.676935671999672, + 2.7503401699977985, + 2.8418701470000087, + 2.948979002998385, + 3.0365244280001207, + 3.1022409689976485, + 3.161470727998676, + 3.2372620809983346, + 3.3289572699977725, + 3.420947475999128, + 3.505814473999635, + 3.601255874000344, + 3.703574438997748, + 3.805380977999448, + 3.9160950999976194, + 4.02890521199879, + 4.135218130999419, + 4.226550953997503, + 4.320650711997587, + 4.40991473199756, + 4.4729822569970565, + 4.553054463998706, + 4.634360183998069, + 4.72688430999915, + 4.840619602997322, + 4.954023548998521, + 5.072793841998646, + 5.175249837997399, + 5.2711158040001465, + 5.328028592997725, + 5.376028654998663, + 5.419600638997508, + 5.474059778000083, + 5.534976700997504, + 5.593613334996917, + 5.652169879998837, + 5.73115600500023, + 5.7985033959994325, + 5.8469268459994055, + 5.893095586998243, + 5.940171618000022, + 6.001165338999272, + 6.079404418997001, + 6.15676745699966, + 6.2291265389976616, + 6.303700978998677, + 6.3798455799988005, + 6.441937129999133, + 6.4857159909988695, + 6.533194758998434, + 6.580955147997884, + 6.632037289997243 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 79.76460500140092, + "p95": 110.71412199817132, + "max": 118.7702930001251 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 77.64128007707482, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.663510667000082, + "end_to_end_tps": 76.83637433576779, + "timing_valid": true, + "steady_tps": 85.19971174804445, + "per_stream_tps": [ + 50.0, + 38.8 + ], + "ttft_s": [ + 0.2, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.20287622199975885, + "decode_tps": 49.9847530331535, + "prefill_tps": 69.00759419709935, + "itl_mean_ms": 20.00610064706587, + "itl_p50_ms": 0.0, + "itl_p95_ms": 96.87835700242431, + "round_gap_p50_ms": 88.32697500110953, + "round_gap_p95_ms": 112.47457200079225, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09449885600042762, + "decode_tps": 38.82064003853741, + "prefill_tps": 148.14994162401973, + "itl_mean_ms": 25.75949286274765, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.19405199980247, + "round_gap_p50_ms": 82.39094599775854, + "round_gap_p95_ms": 109.5541390022845, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 22.88279675490676, + "itl_p50_ms": 0.0, + "itl_p95_ms": 96.87835700242431, + "round_gap_p50_ms": 85.58042900040164, + "round_gap_p95_ms": 112.26407499998459, + "round_gap_mean_ms": 83.95846291368667, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 137.95117263506498 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 29632.791319238, + "first": 29632.99419546, + "last": 29638.095751125, + "end": 29638.095833277, + "elapsed_s": 5.3045140389986045, + "chunk_offsets_s": [ + 0.20287622199975885, + 0.27081929599808063, + 0.3327891879998788, + 0.4014331440012029, + 0.48202298500109464, + 0.5550522250014183, + 0.6310284829996817, + 0.7176432909982395, + 0.810831988001155, + 0.9148722040008579, + 1.0000921030004974, + 1.070771301001514, + 1.1580902130008326, + 1.2345688300010806, + 1.320919010999205, + 1.4111566610008595, + 1.5023129410001275, + 1.5917826840013731, + 1.6809333989986044, + 1.7650667070010968, + 1.8564775839986396, + 1.944054125000548, + 2.039522216000478, + 2.131530696999107, + 2.2355580410003313, + 2.3297932159985066, + 2.4029868280013034, + 2.479865064000478, + 2.552550734999386, + 2.623166598001262, + 2.7015455000000657, + 2.775509440998576, + 2.8610609520001162, + 2.967489384998771, + 3.0558163599998807, + 3.1229387019993737, + 3.187893956001062, + 3.264104074998613, + 3.3537767980014905, + 3.4478211569985433, + 3.533401585998945, + 3.6302799430013692, + 3.7325960810012475, + 3.82903555500161, + 3.935841482998512, + 4.0498008799986565, + 4.159296516998438, + 4.253360927999893, + 4.347832620998815, + 4.438329687000078, + 4.502735873000347, + 4.58440868300022, + 4.66610467099963, + 4.759838802001468, + 4.866298707998794, + 4.9787732799995865, + 5.0973119320005935, + 5.206044896000094, + 5.304431887001556 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 88.7388449991704, + "p95": 112.47457200079225, + "max": 118.538652001007 + } + }, + { + "start": 29632.791225094, + "first": 29632.88572395, + "last": 29639.45439463, + "end": 29639.454735761, + "elapsed_s": 6.663510667000082, + "chunk_offsets_s": [ + 0.09449885600042762, + 0.2706043510006566, + 0.33269175900204573, + 0.4018596380010422, + 0.48195491099977517, + 0.5549216120016354, + 0.6309419499993965, + 0.7175252490014827, + 0.8106727379999938, + 0.914517886001704, + 0.999821202000021, + 1.070485975000338, + 1.1584849560022121, + 1.2346000739999, + 1.3212468010024168, + 1.4110343270003796, + 1.502039359002083, + 1.5915821970011166, + 1.6807712230001925, + 1.7645809720015677, + 1.8569575960027578, + 1.9438534950022586, + 2.0397642470015853, + 2.1319642609996663, + 2.235803854000551, + 2.3300768510016496, + 2.403196208000736, + 2.4801592790026916, + 2.552410274001886, + 2.623387196999829, + 2.7014557239999704, + 2.775976389999414, + 2.861329362000106, + 2.967907054000534, + 3.0561714110008324, + 3.12329782300003, + 3.1877199310001743, + 3.2639148240014038, + 3.354268487000809, + 3.448185975001252, + 3.5335859389997495, + 3.6301391960005276, + 3.73302964100003, + 3.828698829001951, + 3.936127712000598, + 4.050089352000214, + 4.159643491002498, + 4.25315440200211, + 4.348257732002821, + 4.438508512001135, + 4.502614220000396, + 4.58405075400151, + 4.666441699999268, + 4.759605139999621, + 4.866325618000701, + 4.978589693000686, + 5.097170521999942, + 5.2058698649998405, + 5.305063916999643, + 5.3616362790016865, + 5.4105863430013414, + 5.455878626002232, + 5.510631643999659, + 5.57150933199955, + 5.629929087001074, + 5.68844317500043, + 5.767141544001788, + 5.827480563002609, + 5.872482743001456, + 5.9175751360016875, + 5.96341620900057, + 6.022730348002369, + 6.100398042999586, + 6.1828656470024725, + 6.258897602001525, + 6.332827913000074, + 6.407303976000549, + 6.472266053002386, + 6.51724048899996, + 6.563588580000214, + 6.610021770000458, + 6.663169536001078 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 82.39094599775854, + "p95": 109.5541390022845, + "max": 176.10549500022898 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 99.9, + "per_stream_tps": 27.5, + "ttft_s_max": 0.39, + "request": { + "ttft_s": 0.24, + "decode_tps": 27.44, + "prefill_tps": 60.472, + "itl_mean_ms": 36.449, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.704, + "round_gap_p50_ms": 129.9, + "round_gap_p95_ms": 177.957, + "tokens_per_chunk": 3.633 + }, + "pooled": { + "aggregate_prefill_tps": 145.356, + "itl_mean_ms": 35.964, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.222, + "round_gap_p50_ms": 130.797, + "round_gap_p95_ms": 177.958, + "tokens_per_chunk": 3.618 + }, + "timing_valid": true, + "end_to_end_tps": 99.39410812507123, + "makespan_s": 10.302421535001486, + "steady_tps": 110.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 99.94873203278883, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.302421535001486, + "end_to_end_tps": 99.39410812507123, + "timing_valid": true, + "steady_tps": 110.54021964551293, + "per_stream_tps": [ + 31.5, + 25.7, + 27.8, + 26.8 + ], + "ttft_s": [ + 0.29, + 0.39, + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.28720622200125945, + "decode_tps": 31.475443675360612, + "prefill_tps": 48.745462067108726, + "itl_mean_ms": 31.770799176464447, + "itl_p50_ms": 0.0, + "itl_p95_ms": 161.00150900092558, + "round_gap_p50_ms": 139.0506269999605, + "round_gap_p95_ms": 182.12483499883092, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3884505709975201, + "decode_tps": 25.72233067192236, + "prefill_tps": 36.040621497990735, + "itl_mean_ms": 38.87672593726379, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.51939300066442, + "round_gap_p50_ms": 122.07620999834035, + "round_gap_p95_ms": 174.592911000218, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19071003799763275, + "decode_tps": 27.79100793534264, + "prefill_tps": 73.40987473440585, + "itl_mean_ms": 35.98286187843769, + "itl_p50_ms": 0.0, + "itl_p95_ms": 160.98358499948517, + "round_gap_p50_ms": 129.88326599952416, + "round_gap_p95_ms": 177.63718499918468, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.097075641999254, + "decode_tps": 26.7723241893164, + "prefill_tps": 144.21743407174773, + "itl_mean_ms": 37.35200548628699, + "itl_p50_ms": 0.0, + "itl_p95_ms": 160.66565599976457, + "round_gap_p50_ms": 129.91729200075497, + "round_gap_p95_ms": 178.27599399970495, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 35.99559811961323, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.55585099800373, + "round_gap_p50_ms": 129.95973999932176, + "round_gap_p95_ms": 177.9579489993921, + "round_gap_mean_ms": 131.5968103297688, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 144.05421019217397 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 29639.456172116, + "first": 29639.743378338, + "last": 29647.844932128, + "end": 29647.845038762, + "elapsed_s": 8.388866646000679, + "chunk_offsets_s": [ + 0.28720622200125945, + 0.480761552000331, + 0.5666219750019081, + 0.6679857430026459, + 0.7761029140019673, + 0.8645822770013183, + 0.9765613130002748, + 1.0997105099995679, + 1.228530917000171, + 1.3613543570027105, + 1.4830756830015162, + 1.6092033910026657, + 1.7306856750001316, + 1.8364458610012662, + 1.9577002160003758, + 2.0967508430003363, + 2.2560442880021583, + 2.4046013610022783, + 2.55858445800186, + 2.6888848550006514, + 2.8189610230001563, + 2.9668424779993074, + 3.1317840619994968, + 3.2970448980013316, + 3.4750028470007237, + 3.650884598002449, + 3.8023335889993177, + 3.9468766700010747, + 4.07275350800046, + 4.1785194229996705, + 4.295965140001499, + 4.414503015999799, + 4.56621688100131, + 4.724694492000708, + 4.868481999001233, + 4.998712782002258, + 5.1178841090004425, + 5.2470458530006, + 5.396298178002326, + 5.550615821000974, + 5.689268608002749, + 5.851646944000095, + 6.005745034999563, + 6.145528139000817, + 6.294072365002648, + 6.456824811000843, + 6.617826320001768, + 6.761547652000445, + 6.900878439002554, + 7.021674813000573, + 7.123900958002196, + 7.253796428001806, + 7.376095441999496, + 7.508386386001803, + 7.6790680840022105, + 7.8611929190010414, + 8.052766924000025, + 8.227158115001657, + 8.388760011999693 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 139.19070700103475, + "p95": 182.12483499883092, + "max": 193.55532999907155 + } + }, + { + "start": 29639.456369185, + "first": 29639.844819756, + "last": 29649.75838487, + "end": 29649.758498748, + "elapsed_s": 10.302129562998743, + "chunk_offsets_s": [ + 0.3884505709975201, + 0.48068861499996274, + 0.5661646969965659, + 0.6681872429981013, + 0.7762150909984484, + 0.8642872309974337, + 0.9771343169995816, + 1.1006785069985199, + 1.2291514569988067, + 1.3613068549966556, + 1.4829722099966602, + 1.6087074749993917, + 1.730783684997732, + 1.8367861719998473, + 1.9581643040000927, + 2.0966096019983524, + 2.256128994999017, + 2.4050716729980195, + 2.55748765199678, + 2.6885299639980076, + 2.819108009996853, + 2.966929110996716, + 3.131247456996789, + 3.297009978999995, + 3.4751334749998932, + 3.6503222189967346, + 3.8016687169983925, + 3.94728260699776, + 4.0723254369986535, + 4.178032127998449, + 4.296035170998948, + 4.414919813996676, + 4.566552485997818, + 4.725040239998634, + 4.86871909799811, + 4.998467169996729, + 5.117491799999698, + 5.247062920996541, + 5.396383855997556, + 5.5509809059985855, + 5.689599556997564, + 5.851828595998086, + 6.007428486998833, + 6.145655413998611, + 6.294849941998109, + 6.456832659998327, + 6.617786114999035, + 6.761951148997468, + 6.901738228996692, + 7.021816759999638, + 7.1239385529988795, + 7.254129233999265, + 7.375399153999751, + 7.508806410998659, + 7.679240663997916, + 7.861386363998463, + 8.053075782998349, + 8.227668693998567, + 8.388499479999155, + 8.488209739000013, + 8.579883000998962, + 8.66093621699838, + 8.76379666199864, + 8.877156138998544, + 8.984465851997811, + 9.079711600999872, + 9.18817827499879, + 9.281967307997547, + 9.366954191998957, + 9.444725986999401, + 9.529514758996811, + 9.621383783000056, + 9.6869718429989, + 9.764104485999269, + 9.830550121998385, + 9.897501490999275, + 9.972228266997263, + 10.047245386998839, + 10.111382000999583, + 10.157112795997818, + 10.202938630998688, + 10.249734402998001, + 10.302015684999787 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 122.81019999863929, + "p95": 174.592911000218, + "max": 191.68941899988567 + } + }, + { + "start": 29639.456878933, + "first": 29639.647588971, + "last": 29648.82321875, + "end": 29648.823654992, + "elapsed_s": 9.366776058999676, + "chunk_offsets_s": [ + 0.19071003799763275, + 0.47990034299800755, + 0.5657144129982044, + 0.6674483909991977, + 0.7755737059997045, + 0.8641467029992782, + 0.9763367519990425, + 1.0994736860011471, + 1.2283320639980957, + 1.3603863319985976, + 1.482558515999699, + 1.6085849480004981, + 1.730361191999691, + 1.8365600689976418, + 1.9575944479984173, + 2.0959190190005756, + 2.2554748699985794, + 2.4047459219982557, + 2.5573295610010973, + 2.688712127001054, + 2.8185241259998293, + 2.966250649998983, + 3.130636247999064, + 3.296738433000428, + 3.4743756179996126, + 3.65005837499848, + 3.8018915869979537, + 3.946988241001236, + 4.071522426998854, + 4.177864770997985, + 4.295585142001073, + 4.41473387500082, + 4.565398490998632, + 4.724634813999728, + 4.868767677999131, + 4.9976242369994, + 5.117011489997822, + 5.246720954997727, + 5.395948388999386, + 5.54996197500077, + 5.688832636998995, + 5.851086628001212, + 6.006653403997916, + 6.144958893000876, + 6.293915977999859, + 6.456412677998742, + 6.617396262998227, + 6.761060418000852, + 6.900345163998281, + 7.021454401998199, + 7.123511062000034, + 7.253394327999558, + 7.375024408000172, + 7.508521627998562, + 7.67927470900031, + 7.860627318001207, + 8.052523234000546, + 8.226743811999768, + 8.38789546499902, + 8.487450279000768, + 8.579625282000052, + 8.660357797998586, + 8.763496953997674, + 8.877219998998044, + 8.984177375998115, + 9.078952251998999, + 9.188165794999804, + 9.28140128800078, + 9.366339816999243 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 130.63291599974036, + "p95": 177.63718499918468, + "max": 289.1903050003748 + } + }, + { + "start": 29639.456077213, + "first": 29639.553152855, + "last": 29649.077914254, + "end": 29649.077990471, + "elapsed_s": 9.621913258000859, + "chunk_offsets_s": [ + 0.097075641999254, + 0.48078164600156015, + 0.566595268999663, + 0.667792828000529, + 0.7759681540010206, + 0.8643623770003614, + 0.9767109050008003, + 1.0995649390024482, + 1.2286869439994916, + 1.3615169530021376, + 1.483523664999666, + 1.6093361099992762, + 1.730881870000303, + 1.836602868999762, + 1.9580010900026537, + 2.0967983010013995, + 2.2560652690008283, + 2.4049025570020603, + 2.5584936550003476, + 2.6893221520003863, + 2.8188583860028302, + 2.9668333959998563, + 3.131830702001025, + 3.297080645999813, + 3.475356639999518, + 3.651192296001682, + 3.80213002399978, + 3.947403040001518, + 4.072134527999879, + 4.178250659999321, + 4.296013341001526, + 4.414935209002579, + 4.567148301000998, + 4.725065651000477, + 4.868306874999689, + 4.998224167000444, + 5.11770847400112, + 5.246848205999413, + 5.396488640002644, + 5.55103067500022, + 5.689288560002751, + 5.851682470001833, + 6.005972398001177, + 6.145697562002169, + 6.2942168840017985, + 6.4571829910019005, + 6.617848647001665, + 6.76158279000083, + 6.901030678000097, + 7.021715783001127, + 7.123885007000354, + 7.253844746999675, + 7.3764907350014255, + 7.508734054001252, + 7.680190086000948, + 7.861389097000938, + 8.053073244002007, + 8.227201909001451, + 8.38854603800064, + 8.488363730000856, + 8.579921570002625, + 8.660975505001261, + 8.763630191999255, + 8.877102934002323, + 8.984561044999282, + 9.079849614001432, + 9.188381412001036, + 9.281810600001336, + 9.36689131499952, + 9.445261294000375, + 9.5296577170011, + 9.621837041002436 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 129.91729200075497, + "p95": 178.27599399970495, + "max": 383.70600400230614 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 99.94289773315793, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.302682424000523, + "end_to_end_tps": 99.39159122429609, + "timing_valid": true, + "steady_tps": 110.6227656224263, + "per_stream_tps": [ + 30.8, + 25.7, + 28.1, + 27.1 + ], + "ttft_s": [ + 0.1, + 0.38, + 0.29, + 0.2 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09643641200091224, + "decode_tps": 30.787196672681603, + "prefill_tps": 145.17338118995517, + "itl_mean_ms": 32.48103458822965, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.88807499888935, + "round_gap_p50_ms": 137.58068899915088, + "round_gap_p95_ms": 181.72885199965094, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38406904400108033, + "decode_tps": 25.712626242996652, + "prefill_tps": 36.45177922738449, + "itl_mean_ms": 38.891398745095906, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.2224600026384, + "round_gap_p50_ms": 122.30456299948855, + "round_gap_p95_ms": 170.9723669991945, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.29204626999853645, + "decode_tps": 28.11584252433969, + "prefill_tps": 47.93760933865089, + "itl_mean_ms": 35.56713618431697, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.9329070013482, + "round_gap_p50_ms": 133.51903899820172, + "round_gap_p95_ms": 179.69390799771645, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.195622731000185, + "decode_tps": 27.08885266914421, + "prefill_tps": 71.5663252855148, + "itl_mean_ms": 36.91555387058008, + "itl_p50_ms": 0.0, + "itl_p95_ms": 160.61992599861696, + "round_gap_p50_ms": 130.84788099877187, + "round_gap_p95_ms": 179.64191100327298, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 35.96378084705565, + "itl_p50_ms": 0.0, + "itl_p95_ms": 159.2224600026384, + "round_gap_p50_ms": 131.14320100066834, + "round_gap_p95_ms": 179.64191100327298, + "round_gap_mean_ms": 131.48048911826797, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 145.3560352151394 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 29649.760153599, + "first": 29649.856590011, + "last": 29658.139253831, + "end": 29658.13940589, + "elapsed_s": 8.37925229100074, + "chunk_offsets_s": [ + 0.09643641200091224, + 0.4750850150012411, + 0.5558324170015112, + 0.6572471860017686, + 0.7636370770014764, + 0.8525153109985695, + 0.9642382929996529, + 1.0886882970007719, + 1.2227107860016986, + 1.3591282529996533, + 1.4795481129986001, + 1.6006174979993375, + 1.7196991259988863, + 1.8265969280000718, + 1.9486558300013712, + 2.0869920990007813, + 2.2525714920011524, + 2.401694440999563, + 2.54888953200134, + 2.6795925420010462, + 2.808302602999902, + 2.9564136970002437, + 3.123623117000534, + 3.2938418929988984, + 3.4686721910002234, + 3.6377502389987058, + 3.7884513450007944, + 3.933227106001141, + 4.056878309998865, + 4.1650199690011505, + 4.290824955001881, + 4.407663139998476, + 4.551062794002064, + 4.710069543001737, + 4.854319335001492, + 4.9851403499997105, + 5.10549127700142, + 5.238777687001857, + 5.3904822030017385, + 5.538448909999715, + 5.675670533000812, + 5.835558607999701, + 5.991791791999276, + 6.134573456998623, + 6.292171267999947, + 6.453279051998834, + 6.606565051999496, + 6.749577089001832, + 6.887157778000983, + 7.006304831000307, + 7.110141830999055, + 7.244636294999509, + 7.365970109000045, + 7.495404220000637, + 7.661033719999978, + 7.842762571999629, + 8.037080398000398, + 8.21794318100001, + 8.379100231999473 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 137.9584789992805, + "p95": 181.72885199965094, + "max": 378.6486030003289 + } + }, + { + "start": 29649.761042046, + "first": 29650.14511109, + "last": 29660.06241777, + "end": 29660.062532593, + "elapsed_s": 10.301490547000867, + "chunk_offsets_s": [ + 0.38406904400108033, + 0.47436871199897723, + 0.5556850869979826, + 0.6560292949980067, + 0.7629538359979051, + 0.8515812919977179, + 0.9635716060001869, + 1.0877575669983344, + 1.2217730089978431, + 1.3591815899999347, + 1.4789965729978576, + 1.599922729001264, + 1.7190842249983689, + 1.8260515250003664, + 1.9474781600001734, + 2.0860320899992075, + 2.25183120300062, + 2.4010251549989334, + 2.5482534049988317, + 2.6792441329998837, + 2.807666092998261, + 2.955773990997841, + 3.122547527000279, + 3.2935198939994734, + 3.4681192280004325, + 3.6376647819997743, + 3.7883110879993183, + 3.9328685090004, + 4.056209825001133, + 4.164558021999255, + 4.289852025998698, + 4.407098315998155, + 4.550483521998103, + 4.709705982000742, + 4.8537742970002, + 4.9848238570011745, + 5.104476661999797, + 5.23781747799876, + 5.3898357350008155, + 5.537999621999916, + 5.675138022001192, + 5.835006901001179, + 5.991559101999883, + 6.133981773000414, + 6.291450757998973, + 6.453289779998158, + 6.606208483000955, + 6.749837827999727, + 6.88662625500001, + 7.006321807999484, + 7.1083708820005995, + 7.243208359999699, + 7.365512922999187, + 7.494753644998127, + 7.660383688998991, + 7.842357334000553, + 8.03755415699925, + 8.216697723000834, + 8.377609236998978, + 8.473978885998804, + 8.56161487099962, + 8.642335992997687, + 8.746033252999041, + 8.859606070000154, + 8.969161433000409, + 9.066294721000304, + 9.178322765998018, + 9.275732187998074, + 9.360764353001287, + 9.4388849019997, + 9.51825356399786, + 9.607766494998941, + 9.6721814909979, + 9.748052041999472, + 9.816365235998092, + 9.883238817001256, + 9.959276297999168, + 10.033553640998434, + 10.098478782998427, + 10.144862370998453, + 10.196489172998554, + 10.247051304999331, + 10.301375724000536 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 122.8229395001108, + "p95": 170.9723669991945, + "max": 195.196822998696 + } + }, + { + "start": 29649.760452219, + "first": 29650.052498489, + "last": 29659.122118216, + "end": 29659.12224448, + "elapsed_s": 9.36179226099921, + "chunk_offsets_s": [ + 0.29204626999853645, + 0.4752830779980286, + 0.5557935399992857, + 0.657114557998284, + 0.7636263069980487, + 0.8516124519992445, + 0.9639960949971282, + 1.0881578549997357, + 1.2220559870002035, + 1.359596962996875, + 1.4795170149991463, + 1.6005971069971565, + 1.720286039999337, + 1.8269916819990613, + 1.9484442049979407, + 2.086501628997212, + 2.2524586799991084, + 2.401562756000203, + 2.548798869996972, + 2.6799420709976403, + 2.808589726999344, + 2.956457304997457, + 3.122775564999756, + 3.2938753199996427, + 3.4690811359978397, + 3.6376826949999668, + 3.788645483000437, + 3.93392004099951, + 4.057178636998287, + 4.164864836999186, + 4.289615383997443, + 4.407869299997401, + 4.551508174998162, + 4.710600413000066, + 4.854899461999594, + 4.985226148997754, + 5.104820562999521, + 5.238339601997723, + 5.3903424679992895, + 5.53898329599906, + 5.675846759997512, + 5.83577966699886, + 5.992491135999444, + 6.134355344998767, + 6.291540689999238, + 6.453716288997384, + 6.60697321599946, + 6.75003374500011, + 6.887499929998739, + 7.006327267998131, + 7.109606218000408, + 7.2445009339971875, + 7.366394120999757, + 7.495432933999837, + 7.66140349899797, + 7.842714147998777, + 8.03730894399996, + 8.217002851997677, + 8.378699087999848, + 8.47432167399893, + 8.5616467309992, + 8.642878793998534, + 8.74685688399768, + 8.860456137997971, + 8.969377849000011, + 9.066262033000385, + 9.179174872999283, + 9.276506656999118, + 9.361665996999363 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 133.70858549933473, + "p95": 179.69390799771645, + "max": 194.59479600118357 + } + }, + { + "start": 29649.759850169, + "first": 29649.9554729, + "last": 29659.368939137, + "end": 29659.369080108, + "elapsed_s": 9.609229939000215, + "chunk_offsets_s": [ + 0.195622731000185, + 0.4760608350006805, + 0.5568064959988988, + 0.657413816999906, + 0.7640710419982497, + 0.8521323210006813, + 0.9644871210002748, + 1.0888947280000139, + 1.222584144998109, + 1.3596752829980687, + 1.4800541219992738, + 1.600987133999297, + 1.7207080109983508, + 1.8270417659987288, + 1.9485921819978103, + 2.087041737999243, + 2.2531081289998838, + 2.402438909000921, + 2.5496005519999017, + 2.680346165998344, + 2.8089719290001085, + 2.9573468019989377, + 3.1234661849994154, + 3.2942466770000465, + 3.469407777000015, + 3.6386231539981964, + 3.7889846049984044, + 3.934325694997824, + 4.0576502090007125, + 4.1659909570007585, + 4.290491112999007, + 4.408554009998625, + 4.551910415997554, + 4.71069475799959, + 4.855455978999089, + 4.986303859997861, + 5.105498370998248, + 5.238883890000579, + 5.3921654890000354, + 5.539457163999032, + 5.6758081410007435, + 5.8364280669993605, + 5.992822645999695, + 6.134787684000912, + 6.292094325999642, + 6.453946998000902, + 6.607486925997364, + 6.750396332998207, + 6.887895443000161, + 7.007227598998725, + 7.109900817999005, + 7.244618240998534, + 7.367281530998298, + 7.496353509999608, + 7.661907822999638, + 7.843488807997346, + 8.038299461997667, + 8.21794137300094, + 8.379143996000494, + 8.474962677999429, + 8.562596233998192, + 8.643348605000938, + 8.747646478997922, + 8.8606190649989, + 8.970190284999262, + 9.067124605000572, + 9.179462756997964, + 9.276873601000261, + 9.362209509999957, + 9.440031876998546, + 9.519381980997423, + 9.609088967998105 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 130.84788099877187, + "p95": 179.64191100327298, + "max": 280.4381040004955 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 100.20423128348298, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.275749263000762, + "end_to_end_tps": 99.65210066842053, + "timing_valid": true, + "steady_tps": 109.14635470844614, + "per_stream_tps": [ + 31.6, + 25.3, + 28.5, + 26.9 + ], + "ttft_s": [ + 0.28, + 0.19, + 0.38, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.2835340140009066, + "decode_tps": 31.57322283227917, + "prefill_tps": 49.37679187921078, + "itl_mean_ms": 31.672408145095687, + "itl_p50_ms": 0.0, + "itl_p95_ms": 158.0086610010767, + "round_gap_p50_ms": 139.06589399994118, + "round_gap_p95_ms": 182.4166400001559, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18923613299921271, + "decode_tps": 25.28278859775024, + "prefill_tps": 73.98164281901832, + "itl_mean_ms": 39.552599039212936, + "itl_p50_ms": 0.0, + "itl_p95_ms": 157.50455299712485, + "round_gap_p50_ms": 126.2425400018401, + "round_gap_p95_ms": 173.97633899963694, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3763423829987005, + "decode_tps": 28.454906170626753, + "prefill_tps": 37.20016833726735, + "itl_mean_ms": 35.14332445883352, + "itl_p50_ms": 0.0, + "itl_p95_ms": 156.71883200047887, + "round_gap_p50_ms": 129.38523499906296, + "round_gap_p95_ms": 173.58523000075365, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09646619200066198, + "decode_tps": 26.880437607552512, + "prefill_tps": 145.1285648334074, + "itl_mean_ms": 37.201775305883906, + "itl_p50_ms": 0.0, + "itl_p95_ms": 157.01142199759488, + "round_gap_p50_ms": 129.80148700080463, + "round_gap_p95_ms": 174.36033299964038, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 35.89252673725651, + "itl_p50_ms": 0.0, + "itl_p95_ms": 157.01142199759488, + "round_gap_p50_ms": 130.79720699897734, + "round_gap_p95_ms": 174.36033299964038, + "round_gap_mean_ms": 131.21999022222812, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 148.56566798350767 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 29660.063977489, + "first": 29660.347511503, + "last": 29668.42397558, + "end": 29668.42425537, + "elapsed_s": 8.360277880998183, + "chunk_offsets_s": [ + 0.2835340140009066, + 0.4659506540010625, + 0.5460866510002234, + 0.6475582439998107, + 0.7537261180004862, + 0.8435626569989836, + 0.9622873289990821, + 1.088987711998925, + 1.2196487039982458, + 1.3519516859996656, + 1.4721705279989692, + 1.5907621340011247, + 1.710993860000599, + 1.817986469999596, + 1.9497494069983077, + 2.088815300998249, + 2.2448243879989604, + 2.3927366489988344, + 2.536274499998399, + 2.667437797997991, + 2.7936171969995485, + 2.9471299670003646, + 3.1128729289994226, + 3.2774351759981073, + 3.4510651040000084, + 3.6215548219988705, + 3.77249536599993, + 3.9257907299979706, + 4.051911626000219, + 4.157730922001065, + 4.2757331440006965, + 4.3957402919986635, + 4.5370715409990225, + 4.693873887998052, + 4.838389091000863, + 4.976284676999057, + 5.097065100999316, + 5.225501021999662, + 5.37304004500038, + 5.5217171929980395, + 5.660695525999472, + 5.819437982998352, + 5.985976177998964, + 6.127506693999749, + 6.275006382998981, + 6.43621850099953, + 6.588864138000645, + 6.731833936999465, + 6.872382343997742, + 6.998600409999199, + 7.101569179998478, + 7.227654612001061, + 7.349482650999562, + 7.477653107998776, + 7.643535453000368, + 7.825992758000211, + 8.028140409998741, + 8.20198942999923, + 8.359998091000307 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 139.8071504991094, + "p95": 182.4166400001559, + "max": 202.1476519985299 + } + }, + { + "start": 29660.064411059, + "first": 29660.253647192, + "last": 29670.339559947, + "end": 29670.339632151, + "elapsed_s": 10.27522109200072, + "chunk_offsets_s": [ + 0.18923613299921271, + 0.46530754799823626, + 0.5453994099989359, + 0.6472590630000923, + 0.7538066009983595, + 0.8433611840009689, + 0.9621317029996135, + 1.08864711500064, + 1.2194443219996174, + 1.3524226209992776, + 1.470831126000121, + 1.590512448001391, + 1.7107318490016041, + 1.817711738000071, + 1.9492607169995608, + 2.088166318000731, + 2.2445650990011927, + 2.392428729999665, + 2.535737644000619, + 2.666920668001694, + 2.793248948000837, + 2.946370145000401, + 3.1127672420007, + 3.2764477230011835, + 3.4504240620008204, + 3.6214533710008254, + 3.7718976120013394, + 3.9255068759994174, + 4.0517494160012575, + 4.157110161999299, + 4.27507814099954, + 4.394950455000071, + 4.535611520001112, + 4.693116072998237, + 4.838328740999714, + 4.9750822040005005, + 5.09726372699879, + 5.225588217999757, + 5.373316656998213, + 5.521896042999288, + 5.659958615000505, + 5.819131176998781, + 5.985464941000828, + 6.126890707000712, + 6.274300547000166, + 6.435936011999729, + 6.588214400999277, + 6.731326918998093, + 6.871709948998614, + 6.998696721999295, + 7.101371525001014, + 7.22742972999913, + 7.349110938001104, + 7.477390750998893, + 7.643552320998424, + 7.825446679999004, + 8.028010451998853, + 8.202392352999595, + 8.359758982998756, + 8.454773353998462, + 8.542783249999047, + 8.623225078998075, + 8.725651153999934, + 8.838645356001507, + 8.953434815000946, + 9.051485880001565, + 9.161037874000613, + 9.25281664800059, + 9.338099969001632, + 9.415167882998503, + 9.493244448000041, + 9.582874528001412, + 9.648194622001029, + 9.724889516000985, + 9.792095111999515, + 9.86154597299901, + 9.941154945998278, + 10.018485830998543, + 10.083142430001317, + 10.129162585999438, + 10.17661384099847, + 10.223043478999898, + 10.275148887998512 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 126.28541000049154, + "p95": 173.97633899963694, + "max": 276.07141499902355 + } + }, + { + "start": 29660.064478197, + "first": 29660.44082058, + "last": 29669.402368317, + "end": 29669.402536065, + "elapsed_s": 9.33805786800076, + "chunk_offsets_s": [ + 0.3763423829987005, + 0.46495836499889265, + 0.5459951829980128, + 0.6466750879990286, + 0.7535586800004239, + 0.8432430360007857, + 0.9619029969981057, + 1.0889622600006987, + 1.2183474949997617, + 1.351692282998556, + 1.4709140300001309, + 1.5907041979990026, + 1.7110777659981977, + 1.8175632470010896, + 1.9493133789983403, + 2.088213896000525, + 2.2447184570009995, + 2.3924120439987746, + 2.5356221639995056, + 2.6666074079985265, + 2.793262829000014, + 2.9461842749988136, + 3.1125969589993474, + 3.2766947989985056, + 3.4502800289992592, + 3.6214353120012674, + 3.7717327780010237, + 3.9253635770000983, + 4.051861250998627, + 4.156721991999802, + 4.274955449000117, + 4.394232781000028, + 4.535959223998361, + 4.69267805599884, + 4.83799525699942, + 4.97514766400127, + 5.096868739998172, + 5.2244767189986305, + 5.372974220001197, + 5.521469849998539, + 5.6593438210002205, + 5.81927038899812, + 5.985205794000649, + 6.126894584998809, + 6.274274000999867, + 6.435366515997885, + 6.587833540997963, + 6.730863157001295, + 6.871511728000769, + 6.998700584998005, + 7.100508884999726, + 7.2272455929996795, + 7.3481992470005935, + 7.477000947998022, + 7.643294592999155, + 7.825631128998793, + 8.02778506499817, + 8.202384865999193, + 8.359738082999684, + 8.454761773999053, + 8.542296218998672, + 8.623098128999118, + 8.725729333000345, + 8.838987276998523, + 8.953284410999913, + 9.051007636000577, + 9.160703588000615, + 9.252644882999448, + 9.337890120001248 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 130.18523949904193, + "p95": 173.58523000075365, + "max": 202.1539359993767 + } + }, + { + "start": 29660.063882888, + "first": 29660.16034908, + "last": 29669.646801783, + "end": 29669.646926315, + "elapsed_s": 9.583043426999211, + "chunk_offsets_s": [ + 0.09646619200066198, + 0.4662962879992847, + 0.5463398270003381, + 0.6474152880000474, + 0.7537413019999804, + 0.8437514619981812, + 0.962835140999232, + 1.0894762499992794, + 1.219277737000084, + 1.3526039169992146, + 1.4720089250004094, + 1.5916768349998165, + 1.7117953980014136, + 1.8184329420000722, + 1.9496634930001164, + 2.0889970259995607, + 2.244871830000193, + 2.3928966649982613, + 2.536133241999778, + 2.66760571100167, + 2.793588381999143, + 2.947081201000401, + 3.1130413229984697, + 3.277380766998249, + 3.4512196389987366, + 3.6215384130009625, + 3.7727152879997448, + 3.9258064679997915, + 4.05236608999985, + 4.157399089999672, + 4.275680874001409, + 4.395207265999488, + 4.536835790000623, + 4.693847211998218, + 4.838413987999957, + 4.97617536100006, + 5.0973518030004925, + 5.226258082999266, + 5.374069430999953, + 5.521656319000613, + 5.660216460000811, + 5.819437441998161, + 5.985598956998729, + 6.127891243999329, + 6.275046323000424, + 6.436252110001078, + 6.5883183289988665, + 6.731505997999193, + 6.872323974999745, + 6.999091426998348, + 7.100902156998927, + 7.227584459000354, + 7.349143824001658, + 7.4781887470016954, + 7.643759938000585, + 7.826425152998127, + 8.028695066001092, + 8.203055399000732, + 8.359894616998645, + 8.454904174999683, + 8.54252930399889, + 8.623512885998935, + 8.725910689998273, + 8.839490382000804, + 8.95401381299962, + 9.051135175999661, + 9.16141068700017, + 9.253100998001173, + 9.338364717001241, + 9.415430681001453, + 9.493485933999182, + 9.582918895001058 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 129.80148700080463, + "p95": 174.36033299964038, + "max": 369.8300959986227 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 107.5, + "per_stream_tps": 23.7, + "ttft_s_max": 0.48, + "request": { + "ttft_s": 0.28, + "decode_tps": 23.703, + "prefill_tps": 50.054, + "itl_mean_ms": 42.19, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.022, + "round_gap_p50_ms": 158.128, + "round_gap_p95_ms": 194.847, + "tokens_per_chunk": 3.71 + }, + "pooled": { + "aggregate_prefill_tps": 149.388, + "itl_mean_ms": 41.909, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.022, + "round_gap_p50_ms": 158.755, + "round_gap_p95_ms": 195.56, + "tokens_per_chunk": 3.657 + }, + "timing_valid": true, + "end_to_end_tps": 107.02192128785252, + "makespan_s": 11.960166520999337, + "steady_tps": 118.5, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 107.48004408811975, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 11.960166520999337, + "end_to_end_tps": 107.02192128785252, + "timing_valid": true, + "steady_tps": 118.78736069894114, + "per_stream_tps": [ + 26.6, + 21.8, + 24.0, + 23.6, + 23.7 + ], + "ttft_s": [ + 0.19, + 0.29, + 0.38, + 0.48, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.1915055069985101, + "decode_tps": 26.617979677586067, + "prefill_tps": 73.10494731678352, + "itl_mean_ms": 37.56859130980778, + "itl_p50_ms": 0.0, + "itl_p95_ms": 185.73299000127008, + "round_gap_p50_ms": 162.57562399914605, + "round_gap_p95_ms": 205.45474699974875, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28635343199857743, + "decode_tps": 21.844939065837327, + "prefill_tps": 48.89063107184813, + "itl_mean_ms": 45.777193380405045, + "itl_p50_ms": 0.0, + "itl_p95_ms": 185.96286599859013, + "round_gap_p50_ms": 149.35443000285886, + "round_gap_p95_ms": 190.35817300027702, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3820856639977137, + "decode_tps": 23.966405636206815, + "prefill_tps": 36.640997868173805, + "itl_mean_ms": 41.72507196862545, + "itl_p50_ms": 0.0, + "itl_p95_ms": 185.54678200234775, + "round_gap_p50_ms": 158.12760200060438, + "round_gap_p95_ms": 191.55995899927802, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.4752073609997751, + "decode_tps": 23.63261534248041, + "prefill_tps": 29.46082310372003, + "itl_mean_ms": 42.31440259607945, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.12534300255356, + "round_gap_p50_ms": 157.78106200014008, + "round_gap_p95_ms": 190.46281299961265, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09725137900022673, + "decode_tps": 23.72050364743557, + "prefill_tps": 143.956827593852, + "itl_mean_ms": 42.15762088627112, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.99682400215534, + "round_gap_p50_ms": 159.4251919996168, + "round_gap_p95_ms": 196.27852800113033, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 41.90857602823777, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.99682400215534, + "round_gap_p50_ms": 158.7546270020539, + "round_gap_p95_ms": 196.27852800113033, + "round_gap_mean_ms": 154.87952010435697, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 146.98631450146482 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 29670.342642928, + "first": 29670.534148435, + "last": 29680.114139219, + "end": 29680.114353375, + "elapsed_s": 9.77171044699935, + "chunk_offsets_s": [ + 0.1915055069985101, + 0.5736857900010364, + 0.6727533329976723, + 0.7855900030008343, + 0.9020067370001925, + 1.0035495400006766, + 1.1356260499997006, + 1.281633012000384, + 1.4358198949994403, + 1.591746386999148, + 1.7269996579998406, + 1.8622140269981173, + 2.006965754000703, + 2.1413101510006527, + 2.280224531998101, + 2.4353255799978797, + 2.6095886629991583, + 2.7821246679995966, + 2.953554018000432, + 3.111549428998842, + 3.2767583439999726, + 3.4585805040005653, + 3.664214209999045, + 3.854018214999087, + 4.040021852997597, + 4.227738104000309, + 4.395321455998783, + 4.5633851179991325, + 4.724998023000808, + 4.866148894998332, + 5.023943915999553, + 5.179745066998294, + 5.3560473439974885, + 5.541780333998759, + 5.7282708040002035, + 5.886524597997777, + 6.030090617998212, + 6.177981668999564, + 6.356705484999111, + 6.5353544049976335, + 6.704310696997709, + 6.875572057997488, + 7.043958345999272, + 7.206533969998418, + 7.387300974998652, + 7.576435068000137, + 7.757015561997832, + 7.917627236998669, + 8.080997649998608, + 8.239861276000738, + 8.362550743000611, + 8.498383820999152, + 8.642119276999438, + 8.802053807998163, + 8.9878544159983, + 9.184810185997776, + 9.390264932997525, + 9.580270427999494, + 9.771496290999494 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 162.97301849954238, + "p95": 205.45474699974875, + "max": 382.18028300252627 + } + }, + { + "start": 29670.342756159, + "first": 29670.629109591, + "last": 29682.302293903, + "end": 29682.302541786, + "elapsed_s": 11.95978562700111, + "chunk_offsets_s": [ + 0.28635343199857743, + 0.5734891780011822, + 0.6720790260005742, + 0.7856959769997047, + 0.9020058739988599, + 1.0036958220007364, + 1.1360641259998374, + 1.281325439002103, + 1.4360862840003392, + 1.5918579440003668, + 1.7260839109985682, + 1.8627396500014584, + 2.0077110640013416, + 2.141101321998576, + 2.280463424998743, + 2.435878879001393, + 2.609583951001696, + 2.7817890440019255, + 2.9531797480012756, + 3.1115237070007424, + 3.2764231680012017, + 3.4585678779985756, + 3.6641571180007304, + 3.8540532840015658, + 4.040016150000156, + 4.227957626000716, + 4.396044689001428, + 4.563940535001166, + 4.724987731999136, + 4.866099718001351, + 5.023561789999803, + 5.179921453000134, + 5.355794940998749, + 5.54197138499876, + 5.728092047000246, + 5.886972266998782, + 6.029919600001449, + 6.178278458999557, + 6.357348595000076, + 6.535095364000881, + 6.704473146000964, + 6.875335928998538, + 7.042994171999453, + 7.206546738001634, + 7.387418076999893, + 7.575667529999919, + 7.75683704399853, + 7.917603981000866, + 8.081486375001987, + 8.240411834001861, + 8.36236148000171, + 8.49848479299908, + 8.642206033000548, + 8.801333979001356, + 8.987836598000285, + 9.184149535998586, + 9.390517042000283, + 9.580356737998954, + 9.770714910999232, + 9.903108818001783, + 10.03013884000029, + 10.150578535001841, + 10.29346799799896, + 10.442822428001818, + 10.57388639599958, + 10.706725288000598, + 10.846333583001979, + 10.937833888001478, + 11.02156664499853, + 11.097972594001476, + 11.177353981001943, + 11.266579445000389, + 11.332563140000275, + 11.408434598000895, + 11.47685368800012, + 11.544218965002074, + 11.6247795700001, + 11.703832970000803, + 11.768516420001106, + 11.81298436400175, + 11.859789862999605, + 11.907020129001467, + 11.959537744001864 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 152.05763750054757, + "p95": 190.35817300027702, + "max": 287.1357460026047 + } + }, + { + "start": 29670.342451994, + "first": 29670.724537658, + "last": 29681.36443101, + "end": 29681.364617025, + "elapsed_s": 11.022165030997712, + "chunk_offsets_s": [ + 0.3820856639977137, + 0.5736456229969917, + 0.6728691359967343, + 0.7865716259984765, + 0.9023938039972563, + 1.0041473869969195, + 1.1361788409994915, + 1.282219778997387, + 1.4361812049974105, + 1.5920235489975312, + 1.7274866749976354, + 1.8629822409966437, + 2.007789145998686, + 2.1406772079972143, + 2.280325751999044, + 2.4361266039995826, + 2.6098252939991653, + 2.78193422899858, + 2.954856018997816, + 3.112565896997694, + 3.275694260999444, + 3.4575643109965313, + 3.6645377589993586, + 3.85491117499987, + 4.040799705999234, + 4.22776466799769, + 4.3962470019978355, + 4.563502436998533, + 4.7258486339997035, + 4.8659575049969135, + 5.024085106997518, + 5.179762195999501, + 5.356143843997415, + 5.54204891399786, + 5.728803707999759, + 5.8864945719979005, + 6.029641854998772, + 6.178340652997576, + 6.357441808999283, + 6.535836384999129, + 6.704641317999631, + 6.875249591997999, + 7.0435226099980355, + 7.2076691949987435, + 7.386882890998095, + 7.5765112429980945, + 7.757827159999579, + 7.918320061999111, + 8.08170216399958, + 8.240986580000026, + 8.363042654997116, + 8.49897733599937, + 8.642129826999735, + 8.802126175996818, + 8.987672957999166, + 9.184663633997843, + 9.391117357998155, + 9.58033830199929, + 9.771373641997343, + 9.90369092399851, + 10.030403669999941, + 10.151399149999634, + 10.293733850998251, + 10.443055371997616, + 10.573660035999637, + 10.707195287999639, + 10.847066559999803, + 10.938037873998837, + 11.021979015997204 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 158.70600900052523, + "p95": 191.55995899927802, + "max": 206.97344800282735 + } + }, + { + "start": 29670.343402717, + "first": 29670.818610078, + "last": 29681.60878274, + "end": 29681.608922895, + "elapsed_s": 11.26552017799986, + "chunk_offsets_s": [ + 0.4752073609997751, + 0.5727869469992584, + 0.6715157049984555, + 0.7851191939989803, + 0.9019141569988278, + 1.0034528510004748, + 1.1355760959995678, + 1.2811164439990534, + 1.4349218839997775, + 1.5908446479988925, + 1.7257061269992846, + 1.8618156679985987, + 2.0065675859987095, + 2.1407711370011384, + 2.279906404997746, + 2.4355354869985604, + 2.609142420998978, + 2.7812244939996162, + 2.9535703399997146, + 3.1113514019998547, + 3.2754065259978233, + 3.457531869000377, + 3.6635504689984373, + 3.853546648999327, + 4.039636404999328, + 4.22654027200042, + 4.395013540000946, + 4.562978919999296, + 4.724496734997956, + 4.864416835000156, + 5.023259245997906, + 5.178891275998467, + 5.355212260998087, + 5.541515648001223, + 5.728037472999858, + 5.886481607998576, + 6.029622783000377, + 6.177095784998528, + 6.3563312369988125, + 6.5348410830010835, + 6.703735269999015, + 6.874779154000862, + 7.041791809999268, + 7.206980735998513, + 7.386284389998764, + 7.57544681500076, + 7.756638348000706, + 7.917943009000737, + 8.080513446999248, + 8.239407901000959, + 8.361400097001024, + 8.498276841000916, + 8.64170266599831, + 8.800457293000363, + 8.98695359499834, + 9.183388568999362, + 9.389824368998234, + 9.579755303999264, + 9.770218116998876, + 9.902640902000712, + 10.029548951999459, + 10.150764599999093, + 10.292857066000579, + 10.4424237969979, + 10.573187773999962, + 10.706133673000295, + 10.845636287998786, + 10.93701247699937, + 11.020989517000999, + 11.097375700999692, + 11.176805890998367, + 11.265380023000034 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 157.78106200014008, + "p95": 190.46281299961265, + "max": 206.43579999887152 + } + }, + { + "start": 29670.342375265, + "first": 29670.439626644, + "last": 29681.18981997, + "end": 29681.190068987, + "elapsed_s": 10.847693722000258, + "chunk_offsets_s": [ + 0.09725137900022673, + 0.5740187029987283, + 0.6724037359999784, + 0.7860129000000597, + 0.903228045997821, + 1.0042715389972727, + 1.136765369999921, + 1.2820126119986526, + 1.436313111000345, + 1.5918086559977382, + 1.7268797649994667, + 1.862573626996891, + 2.0073996699975396, + 2.140990988998965, + 2.2803362229969935, + 2.435377632999007, + 2.609697570998833, + 2.7819602729978214, + 2.954175151997333, + 3.1121376099981717, + 3.2762442040002497, + 3.457883500999742, + 3.6643588519982586, + 3.8555668919980235, + 4.040563716000179, + 4.227382615998067, + 4.3957048499978555, + 4.564064733996929, + 4.7256182370001625, + 4.865318577998551, + 5.024573595997936, + 5.179661313999532, + 5.356098509997537, + 5.541707497999596, + 5.72832492499947, + 5.887619771998288, + 6.02960081999845, + 6.177795145998971, + 6.356785637999565, + 6.535564248999435, + 6.705063776997122, + 6.875438696999481, + 7.042612337998435, + 7.206518197999685, + 7.386788592997618, + 7.5758885409995855, + 7.757513894997828, + 7.918928118997428, + 8.081419486999948, + 8.240844678999565, + 8.362308247997134, + 8.498778924997168, + 8.642068785997253, + 8.802253800997278, + 8.988181993998296, + 9.184460521999426, + 9.390613760999258, + 9.58064524099973, + 9.77117091099717, + 9.903622221998376, + 10.03037427199888, + 10.151521771000262, + 10.29375197599802, + 10.44302068199977, + 10.574000347998663, + 10.707467402997281, + 10.847444704999361 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 159.8051034998207, + "p95": 196.27852800113033, + "max": 476.7673239985015 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 107.77352558971114, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 11.92523025400078, + "end_to_end_tps": 107.33545371759799, + "timing_valid": true, + "steady_tps": 117.27991927538214, + "per_stream_tps": [ + 26.5, + 21.9, + 23.6, + 23.7, + 24.5 + ], + "ttft_s": [ + 0.09, + 0.28, + 0.19, + 0.47, + 0.37 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.0946550410008058, + "decode_tps": 26.46617848551395, + "prefill_tps": 147.90548767371848, + "itl_mean_ms": 37.78407224705077, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.20692799877725, + "round_gap_p50_ms": 162.18226099954336, + "round_gap_p95_ms": 204.4161490011902, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.27969951200066134, + "decode_tps": 21.897585938428954, + "prefill_tps": 50.05371621802078, + "itl_mean_ms": 45.66713439608244, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.1329220001644, + "round_gap_p50_ms": 147.99026699984097, + "round_gap_p95_ms": 193.82723400121904, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18665690899797482, + "decode_tps": 23.62774745403274, + "prefill_tps": 75.00392069683258, + "itl_mean_ms": 42.32312038824175, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.21850600093603, + "round_gap_p50_ms": 157.95902399986517, + "round_gap_p95_ms": 194.84689200180583, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.46816517299885163, + "decode_tps": 23.706258398360706, + "prefill_tps": 29.903975791967635, + "itl_mean_ms": 42.18295368235547, + "itl_p50_ms": 0.0, + "itl_p95_ms": 181.62371500147856, + "round_gap_p50_ms": 155.3096329989785, + "round_gap_p95_ms": 193.22535900209914, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.37485842499881983, + "decode_tps": 24.45698117461671, + "prefill_tps": 37.347433234411305, + "itl_mean_ms": 40.88812077256187, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.81736400240334, + "round_gap_p50_ms": 158.39503100141883, + "round_gap_p95_ms": 193.51775799805182, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 41.76908029725846, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.12537499857717, + "round_gap_p50_ms": 157.75007399861352, + "round_gap_p95_ms": 194.6491140006401, + "round_gap_mean_ms": 154.3639924029117, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 149.38785424353435 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 29682.304300696, + "first": 29682.398955737, + "last": 29692.03389416, + "end": 29692.034122897, + "elapsed_s": 9.729822200999479, + "chunk_offsets_s": [ + 0.0946550410008058, + 0.5646134669987077, + 0.659570159001305, + 0.7750067910019425, + 0.8920709349986282, + 0.9926576040015789, + 1.1275538080008118, + 1.2726288510020822, + 1.4251983610010939, + 1.5782745590004197, + 1.714496013999451, + 1.852218616000755, + 1.996533575002104, + 2.13051773999905, + 2.270782045001397, + 2.4259118789996137, + 2.5982727000009618, + 2.7740804810018744, + 2.9437485490016115, + 3.1039809079993574, + 3.266163168998901, + 3.447797005999746, + 3.6522131550009362, + 3.842097038999782, + 4.0291377030007425, + 4.217652314000588, + 4.385416959001304, + 4.552422720000322, + 4.711756811000669, + 4.851626401999965, + 5.009026578998601, + 5.164009102001728, + 5.339873058001103, + 5.523802373001672, + 5.706009301000449, + 5.864290097000776, + 6.006001903999277, + 6.154669914001715, + 6.335345313000289, + 6.514436518998991, + 6.680591197000467, + 6.8558063589989615, + 7.025140554000245, + 7.1887798520001525, + 7.367022264999832, + 7.554541348999919, + 7.735480743001972, + 7.8963734880017, + 8.05748916399898, + 8.20563294400199, + 8.325859671000217, + 8.462757084002078, + 8.599407431000145, + 8.763202846999775, + 8.947281960001419, + 9.14193107400206, + 9.34885637700063, + 9.536528048000037, + 9.729593463998754 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 162.91077949972532, + "p95": 204.4161490011902, + "max": 469.95842599790194 + } + }, + { + "start": 29682.304500608, + "first": 29682.58420012, + "last": 29694.229319391, + "end": 29694.229419993, + "elapsed_s": 11.924919385000976, + "chunk_offsets_s": [ + 0.27969951200066134, + 0.564788388001034, + 0.6597317119994841, + 0.7761496960010845, + 0.8919221290016139, + 0.992375293000805, + 1.1275713989998621, + 1.272488743001304, + 1.4254642039995815, + 1.5784491949998483, + 1.7146131150002475, + 1.8521653860007063, + 1.996578590998979, + 2.1304900300019654, + 2.270351692000986, + 2.4257909770021797, + 2.598269825000898, + 2.773947378002049, + 2.942974301000504, + 3.10274374599976, + 3.2671143930019753, + 3.447880817002442, + 3.651197699000477, + 3.8418265120017168, + 4.028980581999349, + 4.217635240001982, + 4.385263356001815, + 4.552333128001919, + 4.711226785002509, + 4.851033596001798, + 5.008298746000946, + 5.1635723340004915, + 5.339578736002295, + 5.523752742999932, + 5.705885665000096, + 5.8639375519996975, + 6.005952786999842, + 6.15472761500132, + 6.334809703999781, + 6.5147546180014615, + 6.680512800001452, + 6.8560549300018465, + 7.024527083001885, + 7.188437684999371, + 7.366760020002403, + 7.554454093999084, + 7.735336352001468, + 7.896004838999943, + 8.057858660999045, + 8.205492838002101, + 8.325833693001186, + 8.462882236999576, + 8.599553423002362, + 8.762698352998996, + 8.947496520999266, + 9.142269997999392, + 9.348240760999033, + 9.536225408999599, + 9.730052643000818, + 9.860602953001944, + 9.98813945800066, + 10.106311194002046, + 10.248899663001794, + 10.396889930001635, + 10.526924370002234, + 10.656842062002397, + 10.801230835000752, + 10.894173978002073, + 10.978870177001227, + 11.055721939999785, + 11.135193317000812, + 11.224702540999715, + 11.29178883500208, + 11.368945255002473, + 11.4383683820015, + 11.506831241000327, + 11.579767363000428, + 11.661647705001087, + 11.730404240999633, + 11.77506615600214, + 11.822338156001933, + 11.87094158599939, + 11.924818783001683 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 148.3825475006597, + "p95": 193.82723400121904, + "max": 285.0888760003727 + } + }, + { + "start": 29682.304360021, + "first": 29682.49101693, + "last": 29693.283412629, + "end": 29693.283549284, + "elapsed_s": 10.979189262998261, + "chunk_offsets_s": [ + 0.18665690899797482, + 0.5641403890003858, + 0.659453587999451, + 0.7760421429993585, + 0.8920999570000276, + 0.9924463700008346, + 1.1272427769981732, + 1.2732370650010125, + 1.4256646920002822, + 1.5785543140009395, + 1.7145284910002374, + 1.8524065060009889, + 1.9967747609989601, + 2.1309364049993746, + 2.270620028000849, + 2.4262594829997397, + 2.598322032001306, + 2.7736682260001544, + 2.943029416997888, + 3.1035411170014413, + 3.2664481709980464, + 3.4480762839993986, + 3.6514206860010745, + 3.8413822849979624, + 4.029188265998528, + 4.218406199001038, + 4.385215700000117, + 4.552569627998309, + 4.7117987720012025, + 4.851096498998231, + 5.009055522998096, + 5.164165189999039, + 5.3399418349981715, + 5.524028853000345, + 5.706247359001281, + 5.863997432999895, + 6.006580855999346, + 6.154792085999361, + 6.3357699689986475, + 6.51509843399981, + 6.680799975998525, + 6.856258376999904, + 7.0251615829984075, + 7.188527980000799, + 7.3672770189987205, + 7.5542477070011955, + 7.735068080000929, + 7.896424276998005, + 8.058042870998179, + 8.205405296997924, + 8.326181092001207, + 8.463459620001231, + 8.599808570001187, + 8.76300409599935, + 8.947779156998877, + 9.142626049000683, + 9.348723142000381, + 9.536259255000914, + 9.729703933000565, + 9.860842204998335, + 9.987503257998469, + 10.10592607199942, + 10.249229838998872, + 10.396963762999803, + 10.526975707998645, + 10.656681914999353, + 10.800941618999786, + 10.89399043999947, + 10.97905260799962 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 158.5940840013791, + "p95": 194.84689200180583, + "max": 377.48348000241094 + } + }, + { + "start": 29682.30460349, + "first": 29682.772768663, + "last": 29693.529421852, + "end": 29693.529526357, + "elapsed_s": 11.224922867000714, + "chunk_offsets_s": [ + 0.46816517299885163, + 0.5645882200005872, + 0.6593332619995635, + 0.7750697480005329, + 0.8923651619988959, + 0.992649979001726, + 1.1273480449999624, + 1.2727032239999971, + 1.4251417730010871, + 1.578211640000518, + 1.7146676759984985, + 1.852257301001373, + 1.996390287000395, + 2.1306249599983857, + 2.270164156001556, + 2.4254737890005345, + 2.5984917620007764, + 2.773579755001265, + 2.943211411999073, + 3.1029206839994004, + 3.2665523709983972, + 3.448176085999876, + 3.6517367500018736, + 3.8413901739986613, + 4.028753771999618, + 4.217601950000244, + 4.385027639000327, + 4.552178771999024, + 4.711498551998375, + 4.851045128001715, + 5.008602383000834, + 5.163592501001403, + 5.3396602280008665, + 5.523710738001682, + 5.7058361130002595, + 5.863504617998842, + 6.006089864000387, + 6.1546864100018865, + 6.33534866500122, + 6.515120559000934, + 6.6805907919988385, + 6.85697572600111, + 7.0246134379995055, + 7.188667719001387, + 7.36736302799909, + 7.553900408998743, + 7.735333077998803, + 7.896117595999385, + 8.057030184998439, + 8.204925881000236, + 8.326074186999904, + 8.463027903999318, + 8.599296051001147, + 8.763123353000992, + 8.947293662000448, + 9.14225946499937, + 9.348306249001325, + 9.535952460999397, + 9.729177820001496, + 9.860194991000753, + 9.987693852999655, + 10.10647015900031, + 10.248722683001688, + 10.396662580998964, + 10.52690157500183, + 10.656271569001547, + 10.801070580000669, + 10.894098454999039, + 10.978556854999624, + 11.055831764999311, + 11.135510705000343, + 11.224818361999496 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 155.3096329989785, + "p95": 193.22535900209914, + "max": 206.04678400195553 + } + }, + { + "start": 29682.304189739, + "first": 29682.679048164, + "last": 29693.105518961, + "end": 29693.106078217, + "elapsed_s": 10.801888477999455, + "chunk_offsets_s": [ + 0.37485842499881983, + 0.5644190540006093, + 0.6599012250007945, + 0.7756190820000484, + 0.8923651430013706, + 0.9929675790008332, + 1.1272938500005694, + 1.2731841669992718, + 1.4250164209988725, + 1.5786698930023704, + 1.7147626869991655, + 1.8519419459989876, + 1.9964554270009103, + 2.130407626998931, + 2.2705185820013867, + 2.4255610600012005, + 2.598658457001875, + 2.773604037000041, + 2.942871794999519, + 3.102873025000008, + 3.267146785001387, + 3.448394910999923, + 3.65238244700231, + 3.8423050439996587, + 4.029492630001187, + 4.218118729000707, + 4.3853206599997065, + 4.553047061999678, + 4.7114420930010965, + 4.851609576999181, + 5.0087917179989745, + 5.164252974998817, + 5.340219159999833, + 5.5238325919999625, + 5.706649956002366, + 5.863665409000532, + 6.0061945429988555, + 6.155181507001544, + 6.3347323939997295, + 6.514629557001172, + 6.680926938999619, + 6.856001907999598, + 7.025104682001256, + 7.189173098002357, + 7.367183626000042, + 7.553962788999343, + 7.736028767001699, + 7.895798909001314, + 8.05766032700194, + 8.205129492998822, + 8.325801315000717, + 8.463832812001783, + 8.599653608998779, + 8.763456828000926, + 8.947175133998826, + 9.142159358001663, + 9.348828244001197, + 9.536960091001674, + 9.730477848999726, + 9.86068441200041, + 9.988328441999329, + 10.106357241002115, + 10.248888443999022, + 10.396579855001619, + 10.527021681002225, + 10.656783313999767, + 10.801329222002096 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 159.0825865005172, + "p95": 193.51775799805182, + "max": 206.6688859995338 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 107.27851278753347, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 11.980363957001828, + "end_to_end_tps": 106.84149534972302, + "timing_valid": true, + "steady_tps": 118.474173901131, + "per_stream_tps": [ + 27.4, + 22.0, + 23.7, + 23.0, + 23.7 + ], + "ttft_s": [ + 0.46, + 0.37, + 0.28, + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.4645591879998392, + "decode_tps": 27.359926369119144, + "prefill_tps": 30.136095381682228, + "itl_mean_ms": 36.5498059647079, + "itl_p50_ms": 0.0, + "itl_p95_ms": 182.6760039984947, + "round_gap_p50_ms": 163.29141199821606, + "round_gap_p95_ms": 195.877162997931, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3731247790019552, + "decode_tps": 21.969748484124228, + "prefill_tps": 37.520960246723895, + "itl_mean_ms": 45.517134650977894, + "itl_p50_ms": 0.0, + "itl_p95_ms": 185.03381700065802, + "round_gap_p50_ms": 148.49774400136084, + "round_gap_p95_ms": 193.39716100148507, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.27912277500217897, + "decode_tps": 23.70251839188191, + "prefill_tps": 50.15713963108424, + "itl_mean_ms": 42.189609705882525, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.67004700141842, + "round_gap_p50_ms": 160.50549699866679, + "round_gap_p95_ms": 196.87844600048265, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18676197799868532, + "decode_tps": 22.980896318125023, + "prefill_tps": 74.9617248115596, + "itl_mean_ms": 43.51440370980223, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.02183600119315, + "round_gap_p50_ms": 156.6801429980842, + "round_gap_p95_ms": 196.10002400077065, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09533324199946946, + "decode_tps": 23.682718270167626, + "prefill_tps": 146.85328754557526, + "itl_mean_ms": 42.224882658831795, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.43760800073505, + "round_gap_p50_ms": 160.62447399963276, + "round_gap_p95_ms": 196.0965270009183, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 41.99916733804047, + "itl_p50_ms": 0.0, + "itl_p95_ms": 184.02183600119315, + "round_gap_p50_ms": 160.51350099951378, + "round_gap_p95_ms": 195.56047700098134, + "round_gap_mean_ms": 155.21431407536696, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 150.2946054454176 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 29694.232946599, + "first": 29694.697505787, + "last": 29704.017706308, + "end": 29704.017883074, + "elapsed_s": 9.784936474999995, + "chunk_offsets_s": [ + 0.4645591879998392, + 0.5612366929999553, + 0.6540746739992755, + 0.7714914429998316, + 0.89131474199894, + 0.995435703000112, + 1.1289116730004025, + 1.275456691000727, + 1.427809636999882, + 1.580304414001148, + 1.7116581679983938, + 1.8552163319982355, + 1.9993008530000225, + 2.13270496000041, + 2.272720097000274, + 2.430010312000377, + 2.602822451997781, + 2.7763981589996547, + 2.948144015001162, + 3.1089381769998, + 3.273615139998583, + 3.4583727860008366, + 3.659392409997963, + 3.854748171997926, + 4.042934502998833, + 4.230513337999582, + 4.397409917000914, + 4.566716362998704, + 4.7216267849980795, + 4.867551989998901, + 5.028120049999416, + 5.184799148999446, + 5.363458203999471, + 5.548811226999533, + 5.7307930780007155, + 5.892967831998249, + 6.035871259999112, + 6.185996186999546, + 6.365138179000496, + 6.547407393998583, + 6.710695827998279, + 6.889424451001105, + 7.059128402001079, + 7.225519241997972, + 7.405448235000222, + 7.592710312001145, + 7.77538631599964, + 7.938677727997856, + 8.102484925999306, + 8.25054381299924, + 8.371673539000767, + 8.508985802000097, + 8.648277677999431, + 8.812602890000562, + 8.997993950000819, + 9.19387111299875, + 9.400828374000412, + 9.59112865499992, + 9.784759709000355 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 163.5493049998331, + "p95": 195.877162997931, + "max": 206.95726100166212 + } + }, + { + "start": 29694.232045233, + "first": 29694.605170012, + "last": 29706.212039348, + "end": 29706.212117831, + "elapsed_s": 11.980072598002153, + "chunk_offsets_s": [ + 0.3731247790019552, + 0.5619017240023823, + 0.6547830359995714, + 0.7726167560031172, + 0.8921141990031174, + 0.9957629100026679, + 1.1287992750003468, + 1.2758489980005834, + 1.4283723590015143, + 1.5812475420025294, + 1.7126713110010314, + 1.855579872000817, + 1.9997408170020208, + 2.1329471339995507, + 2.273882200002845, + 2.4306183670014434, + 2.604067804000806, + 2.7771395670024503, + 2.9485377439996228, + 3.1093296899998677, + 3.2740704370007734, + 3.4591042540014314, + 3.6598405490003643, + 3.8551520640030503, + 4.042814784002985, + 4.2306387910030026, + 4.398939539001731, + 4.567383458001132, + 4.722828778001713, + 4.868227159000526, + 5.028757145002601, + 5.186235188000865, + 5.363745329002995, + 5.549492736001412, + 5.731793189999735, + 5.894250362001912, + 6.037515069001529, + 6.186954894001246, + 6.366168205000577, + 6.547798663999856, + 6.711777336000523, + 6.889729458001966, + 7.0598488570030895, + 7.225822276002873, + 7.40573374700034, + 7.593334686000162, + 7.776588642002025, + 7.939367443999799, + 8.102818062001461, + 8.250813758000731, + 8.37222820900206, + 8.5095559040019, + 8.64890565800306, + 8.81359780099956, + 8.998815725000895, + 9.194147078000242, + 9.401317928000935, + 9.591933715000778, + 9.785330876002263, + 9.915934136002761, + 10.045135484000639, + 10.164328791001026, + 10.307176054000593, + 10.455673798001953, + 10.586785368002893, + 10.71431322600256, + 10.861868362000678, + 10.953904006000812, + 11.038212208000914, + 11.115335940001387, + 11.194067374999577, + 11.28409850600292, + 11.348200331001863, + 11.425047325999913, + 11.493815770001675, + 11.560705563002557, + 11.63621362900085, + 11.716491074002988, + 11.78530757200133, + 11.831534958000702, + 11.878477570000541, + 11.926362501002586, + 11.979994115001318 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 148.9687845005392, + "p95": 193.39716100148507, + "max": 207.17085000069346 + } + }, + { + "start": 29694.232308967, + "first": 29694.511431742, + "last": 29705.269782217, + "end": 29705.269889526, + "elapsed_s": 11.037580559001071, + "chunk_offsets_s": [ + 0.27912277500217897, + 0.5615775010010111, + 0.6544615120001254, + 0.7725524840025173, + 0.8921122329993523, + 0.9962421109994466, + 1.1292268069992133, + 1.276012192000053, + 1.428205746000458, + 1.5810335530004522, + 1.712477608001791, + 1.8555349210000713, + 2.0001917379995575, + 2.132938467999338, + 2.274045752001257, + 2.4298443700026837, + 2.603686443002516, + 2.7764499980003166, + 2.948505620002834, + 3.1090111170015007, + 3.274568785000156, + 3.4592388320015743, + 3.65979218999928, + 3.8544136700002127, + 4.043213086002652, + 4.2307599390005635, + 4.39818979499978, + 4.566760942001565, + 4.7222063420013, + 4.868059477001225, + 5.029039159999229, + 5.186020219000056, + 5.3635930080017715, + 5.549177156000951, + 5.731482659000903, + 5.893758180001896, + 6.037416777002363, + 6.186365620000288, + 6.365727521999361, + 6.547824040000705, + 6.71172355499948, + 6.890333441999246, + 7.059640544001013, + 7.225800731001073, + 7.405715635999513, + 7.59371212699989, + 7.7758465170009, + 7.939058807001857, + 8.102795721999428, + 8.250908640002308, + 8.3725477040025, + 8.509954248002032, + 8.64849255900117, + 8.812995270000101, + 8.997799762000795, + 9.194678208001278, + 9.401408390000142, + 9.591437179002241, + 9.785255841001344, + 9.915540369001974, + 10.044732880000083, + 10.163615748002485, + 10.306656938002561, + 10.455189579002763, + 10.586877410001762, + 10.713834763002524, + 10.861368556001253, + 10.953572074002295, + 11.037473250002222 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 160.74258999833546, + "p95": 196.87844600048265, + "max": 282.45472599883215 + } + }, + { + "start": 29694.232649374, + "first": 29694.419411352, + "last": 29705.515584298, + "end": 29705.515694493, + "elapsed_s": 11.283045118998416, + "chunk_offsets_s": [ + 0.18676197799868532, + 0.5617390519983019, + 0.6547372450004332, + 0.7720932359989092, + 0.8918988289988192, + 0.9954213880009775, + 1.1289744429996063, + 1.276002411999798, + 1.4272408440010622, + 1.5809434590009914, + 1.7123702309982036, + 1.8557976170013717, + 2.000052429000789, + 2.133111375998851, + 2.2739812770014396, + 2.429108013999212, + 2.602411599000334, + 2.7761963379998633, + 2.9482587040001818, + 3.1092924090007728, + 3.2739627880000626, + 3.4579846240012557, + 3.659193139999843, + 3.8547536170008243, + 4.0423585239987005, + 4.23116972400021, + 4.398122094000428, + 4.566517982999358, + 4.722306258998287, + 4.867538410999259, + 5.028645847000007, + 5.185325989998091, + 5.363691528000345, + 5.549203475999093, + 5.730961869001476, + 5.893171286999859, + 6.036425739999686, + 6.186539657999674, + 6.3651378370013845, + 6.547612548998586, + 6.7114437199998065, + 6.890146942001593, + 7.059545004998654, + 7.2260529279992625, + 7.40563872599887, + 7.593113016999268, + 7.775927853999747, + 7.939077658000315, + 8.102705707999121, + 8.250749643000745, + 8.372728793998249, + 8.509187652998662, + 8.648624467001355, + 8.813050938999368, + 8.998001282998302, + 9.194101306999073, + 9.40097942599823, + 9.591553843998554, + 9.78442452300078, + 9.91578976099845, + 10.045068282001012, + 10.163634822998574, + 10.306650530001207, + 10.455166539999482, + 10.58673624199946, + 10.713408968000294, + 10.861483681001118, + 10.953542479001044, + 11.037826452000445, + 11.114540914000827, + 11.193765659998462, + 11.282934923998255 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 156.6801429980842, + "p95": 196.10002400077065, + "max": 374.97707399961655 + } + }, + { + "start": 29694.231753874, + "first": 29694.327087116, + "last": 29705.094432194, + "end": 29705.094585861, + "elapsed_s": 10.86283198700039, + "chunk_offsets_s": [ + 0.09533324199946946, + 0.5627702790006879, + 0.6551141750023817, + 0.7727989930026524, + 0.8927361050009495, + 0.9967409080018115, + 1.1293502650005394, + 1.2768502600010834, + 1.4285392050005612, + 1.5814021799997136, + 1.7129197680005745, + 1.8556213520023448, + 2.0002389610017417, + 2.1336297199995897, + 2.27468425600091, + 2.4298179159995925, + 2.6038197790003323, + 2.7771758070011856, + 2.949245271000109, + 3.1098697449997417, + 3.2745154420008475, + 3.4592586510007095, + 3.6599690959992586, + 3.8546886059993994, + 4.043626447000861, + 4.231080760000623, + 4.39892696500101, + 4.567371717999777, + 4.722871292000491, + 4.8685870740009705, + 5.029100575000484, + 5.186402565999742, + 5.364430461999291, + 5.549829777002742, + 5.7321775810014515, + 5.894748439000978, + 6.0376787160021195, + 6.1871359210017545, + 6.366524414999731, + 6.5481603469997935, + 6.712022612002329, + 6.88988457600135, + 7.060688822999509, + 7.226522536002449, + 7.406831850999879, + 7.593682676000753, + 7.777115537999634, + 7.93944426500093, + 8.10306235200187, + 8.25135162800143, + 8.373359365999931, + 8.51044433700008, + 8.649157301999367, + 8.813649763000285, + 8.99808737100102, + 9.194183898001938, + 9.401361789001385, + 9.592067473000498, + 9.78553627400106, + 9.916169170999638, + 10.045152937000239, + 10.164403803002642, + 10.30741375100115, + 10.455789416999323, + 10.587180604001333, + 10.714546376999351, + 10.862678320001578 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 161.47660050046397, + "p95": 196.0965270009183, + "max": 467.4370370012184 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.4, + "per_stream_tps": 42.4, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.082, + "decode_tps": 42.426, + "prefill_tps": null, + "itl_mean_ms": 23.57, + "itl_p50_ms": 0.0, + "itl_p95_ms": 64.828, + "round_gap_p50_ms": 53.725, + "round_gap_p95_ms": 71.115, + "tokens_per_chunk": 2.349 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 23.57, + "itl_p50_ms": 0.0, + "itl_p95_ms": 64.828, + "round_gap_p50_ms": 53.725, + "round_gap_p95_ms": 71.115, + "tokens_per_chunk": 2.349 + }, + "timing_valid": true, + "end_to_end_tps": 42.00711809336281, + "makespan_s": 6.094205258999864, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.426013938965916, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.094205258999864, + "end_to_end_tps": 42.00711809336281, + "timing_valid": true, + "per_stream_tps": [ + 42.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08341980000113836, + "decode_tps": 42.426013938965916, + "prefill_tps": null, + "itl_mean_ms": 23.57044433725498, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.82763500045985, + "round_gap_p50_ms": 53.72528099906049, + "round_gap_p95_ms": 70.82532200001879, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.57044433725498, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.82763500045985, + "round_gap_p50_ms": 53.72528099906049, + "round_gap_p95_ms": 70.82532200001879, + "round_gap_mean_ms": 55.6524380185187, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 29787.563957952, + "first": 29787.647377752, + "last": 29793.657841058, + "end": 29793.658163211, + "elapsed_s": 6.094205258999864, + "chunk_offsets_s": [ + 0.08341980000113836, + 0.1407189500023378, + 0.19903744800103595, + 0.25791782600208535, + 0.31054831499932334, + 0.3595001920002687, + 0.41741481899953214, + 0.4677757930003281, + 0.5205484440011787, + 0.5814155480002228, + 0.6434849910001503, + 0.6884072239990928, + 0.748980765001761, + 0.8092140650005604, + 0.8591832050005905, + 0.9071964110007684, + 0.9551044120016741, + 1.000212744998862, + 1.044580853998923, + 1.0929200810023758, + 1.141427697999461, + 1.1931605720019434, + 1.2468858530010039, + 1.2980277550013852, + 1.348666639001749, + 1.4078525619988795, + 1.4704508249997161, + 1.5336585100012599, + 1.6025489890016615, + 1.6552070719990297, + 1.7064370110019809, + 1.7547816469996178, + 1.8068592109993915, + 1.862249021000025, + 1.9231759579997743, + 1.9792230669991113, + 2.035634238000057, + 2.1079970550017606, + 2.1745135269993625, + 2.2217896940019273, + 2.2731906730004994, + 2.321574514000531, + 2.3671671400006744, + 2.4235394040006213, + 2.4808638870017603, + 2.5361408770004346, + 2.591006112001196, + 2.6539812539995182, + 2.7132264589999977, + 2.7626366570002574, + 2.8093208089994732, + 2.8655131270024867, + 2.935854596998979, + 2.997710751998966, + 3.05064570399918, + 3.1033427139991545, + 3.1572375710020424, + 3.2162554659989837, + 3.274372621999646, + 3.32445541999914, + 3.3701146679995873, + 3.4281285670003854, + 3.481188628000382, + 3.528079424999305, + 3.585752331000549, + 3.638551603999076, + 3.683094140000321, + 3.7297574869990058, + 3.777176554001926, + 3.8346129460005614, + 3.890213173999655, + 3.9361669630015967, + 3.9810087000005296, + 4.030366121001862, + 4.089318772999832, + 4.172226047001459, + 4.254450037002243, + 4.332436026001233, + 4.400479667001491, + 4.452953631000128, + 4.501000734999252, + 4.546600308000052, + 4.6075483550011995, + 4.678070622001542, + 4.743505654001638, + 4.797621932000766, + 4.842759900999226, + 4.902790775999165, + 4.968263466998906, + 5.028905688999657, + 5.089021489999141, + 5.136475677001727, + 5.185501150001073, + 5.2329469280011836, + 5.280353237001691, + 5.327592096000444, + 5.375698446001479, + 5.443610993999755, + 5.514436315999774, + 5.564825049001229, + 5.616524312001275, + 5.663924617001612, + 5.723253833999479, + 5.801822584999172, + 5.865652586002398, + 5.927757967001526, + 5.9935856020019855, + 6.040535470001487, + 6.093883106001158 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 53.81006900097418, + "p95": 70.82532200001879, + "max": 82.90727400162723 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.3722021262881, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.099819180999475, + "end_to_end_tps": 41.96845716302914, + "timing_valid": true, + "per_stream_tps": [ + 42.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08139976899838075, + "decode_tps": 42.3722021262881, + "prefill_tps": null, + "itl_mean_ms": 23.600378309806818, + "itl_p50_ms": 0.0, + "itl_p95_ms": 64.67154799975106, + "round_gap_p50_ms": 53.98789500031853, + "round_gap_p95_ms": 71.1152209987631, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.600378309806818, + "itl_p50_ms": 0.0, + "itl_p95_ms": 64.67154799975106, + "round_gap_p50_ms": 53.98789500031853, + "round_gap_p95_ms": 71.1152209987631, + "round_gap_mean_ms": 55.72311545371054, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 29793.658847865, + "first": 29793.740247634, + "last": 29799.758344103, + "end": 29799.758667046, + "elapsed_s": 6.099819180999475, + "chunk_offsets_s": [ + 0.08139976899838075, + 0.13898922199950903, + 0.19896083699859446, + 0.25528784199923393, + 0.3118349219985248, + 0.36389676699764095, + 0.4165299109990883, + 0.4640712629989139, + 0.5158633639985055, + 0.5782795560007798, + 0.6404660329972103, + 0.685755614998925, + 0.746347075997619, + 0.8049678959978337, + 0.8552121499997156, + 0.9026811629992153, + 0.9500626730005024, + 0.9956410519989731, + 1.0407252570003038, + 1.0886703979995218, + 1.1390377419993456, + 1.1905533589997503, + 1.2474505789978139, + 1.2974970809991646, + 1.3545023439983197, + 1.4128864669983159, + 1.4720928389979235, + 1.5323167969982023, + 1.6008963879976363, + 1.6523873959995399, + 1.7038192269974388, + 1.7508972549985629, + 1.8029201680001279, + 1.8579745739989448, + 1.917483908997383, + 1.973617624000326, + 2.0307424089987762, + 2.1018576299975393, + 2.1689232040007482, + 2.2175250859982043, + 2.268860265998228, + 2.323672368998814, + 2.3705961179985025, + 2.419669504997728, + 2.4743193379981676, + 2.5306017199982307, + 2.5845896149985492, + 2.6464392699999735, + 2.7066070129985746, + 2.755132810998475, + 2.8037281449978764, + 2.860356506000244, + 2.9308506439992925, + 2.9935988929973973, + 3.0468847269985417, + 3.0993459040000744, + 3.152285361000395, + 3.2119411089988716, + 3.268879077000747, + 3.324068776000786, + 3.3718523059978907, + 3.4245010150007147, + 3.4737344389977807, + 3.5203112449999026, + 3.5794294019979134, + 3.634005124000396, + 3.6796925849994295, + 3.7267035039985785, + 3.7726914849990862, + 3.832491664998088, + 3.8892003439978, + 3.933925948000251, + 3.9797921040008077, + 4.0296932220007875, + 4.088302239997574, + 4.16989318099877, + 4.25269933799791, + 4.336288886999682, + 4.406056501997227, + 4.45328770299966, + 4.500794203999249, + 4.546914422997361, + 4.607845413000177, + 4.681822757000191, + 4.745585149998078, + 4.79922055499992, + 4.844588504998683, + 4.904724992000411, + 4.969396540000162, + 5.030025492000277, + 5.091511324997555, + 5.137798128998838, + 5.186652636999497, + 5.236281625999254, + 5.282018453999626, + 5.334725097000046, + 5.385386539997853, + 5.449269872999139, + 5.517547064999235, + 5.569362063997687, + 5.620247858998482, + 5.668096024997794, + 5.728043360999436, + 5.806711474000622, + 5.8712608339992585, + 5.934378491998359, + 6.000328109999828, + 6.047435128999496, + 6.099496237999119 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 54.28180850140052, + "p95": 71.1152209987631, + "max": 83.58954900177196 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.49375205154959, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.0830605260016455, + "end_to_end_tps": 42.084079043064705, + "timing_valid": true, + "per_stream_tps": [ + 42.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08187678700051038, + "decode_tps": 42.49375205154959, + "prefill_tps": null, + "itl_mean_ms": 23.53287134510716, + "itl_p50_ms": 0.0, + "itl_p95_ms": 64.82788499852177, + "round_gap_p50_ms": 53.247252999426564, + "round_gap_p95_ms": 71.84769100058475, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.53287134510716, + "itl_p50_ms": 0.0, + "itl_p95_ms": 64.82788499852177, + "round_gap_p50_ms": 53.247252999426564, + "round_gap_p95_ms": 71.84769100058475, + "round_gap_mean_ms": 55.5637240092808, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 29799.759375469, + "first": 29799.841252256, + "last": 29805.842134449, + "end": 29805.842435995, + "elapsed_s": 6.0830605260016455, + "chunk_offsets_s": [ + 0.08187678700051038, + 0.14016524299950106, + 0.20068361400262802, + 0.26187165600276785, + 0.3148257930006366, + 0.3628994339997007, + 0.41329293500166386, + 0.45975031000125455, + 0.511484480000945, + 0.5723460270019132, + 0.6327781840009266, + 0.6769248410018918, + 0.7365254670003196, + 0.7952726110015647, + 0.8458575130025565, + 0.8935076390007453, + 0.9407240360014839, + 0.9850806810027279, + 1.029011366001214, + 1.0770705210015876, + 1.1267487100012659, + 1.1762408310023602, + 1.2360468780025258, + 1.2886667720013065, + 1.3401144849995035, + 1.3961451270006364, + 1.4537351809995016, + 1.513780912002403, + 1.5812403290001384, + 1.6341954700001224, + 1.6843951959999686, + 1.732707451999886, + 1.7837101069999335, + 1.83784849100266, + 1.8968295800004853, + 1.9546612390004157, + 2.0136536800018803, + 2.08658028300124, + 2.154742778002401, + 2.2079900310018274, + 2.2631238360008865, + 2.3140257459999702, + 2.3588724300025206, + 2.408202826001798, + 2.461302757001249, + 2.5158353820006596, + 2.569424651002919, + 2.630555263000133, + 2.689190969002084, + 2.7386703040028806, + 2.7857953140010068, + 2.8430389919994923, + 2.9129343460008386, + 2.9750340589998814, + 3.027822215000924, + 3.0791355300025316, + 3.134421050999663, + 3.192127807000361, + 3.253464150002401, + 3.305887309001264, + 3.351986781002779, + 3.405091947002802, + 3.4549300460021186, + 3.5008098979997158, + 3.5581572400005825, + 3.6138221390028775, + 3.6598565000022063, + 3.7093590870026674, + 3.756515188000776, + 3.8141385820017604, + 3.871038393001072, + 3.9170994580017577, + 3.962022676001652, + 4.011104004002846, + 4.071248976000788, + 4.153767249001248, + 4.241337456001929, + 4.320315641001798, + 4.38707604999945, + 4.434569196000666, + 4.480709358002059, + 4.526239163002174, + 4.586843969002075, + 4.65869166000266, + 4.7235195450011815, + 4.7766419390027295, + 4.822267512001417, + 4.884792209002626, + 4.949465156001679, + 5.012447496999812, + 5.073117829000694, + 5.121659639000427, + 5.17072384399944, + 5.223777417002566, + 5.271581313001661, + 5.320328329002223, + 5.367869162000716, + 5.4295456940017175, + 5.497713176002435, + 5.54794925600072, + 5.601975404002587, + 5.648800145001587, + 5.708328849999816, + 5.785340010999789, + 5.849801329000911, + 5.91187849200287, + 5.977750396999909, + 6.024453101003019, + 6.082758980002836 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 53.418261000842904, + "p95": 71.84769100058475, + "max": 87.57020700068097 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 61.3, + "per_stream_tps": 31.3, + "ttft_s_max": 0.19, + "request": { + "ttft_s": 0.123, + "decode_tps": 31.258, + "prefill_tps": null, + "itl_mean_ms": 31.994, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.315, + "round_gap_p50_ms": 75.033, + "round_gap_p95_ms": 92.686, + "tokens_per_chunk": 2.349 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 31.99, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.974, + "round_gap_p50_ms": 74.61, + "round_gap_p95_ms": 93.18, + "tokens_per_chunk": 2.349 + }, + "timing_valid": true, + "end_to_end_tps": 60.96175665284411, + "makespan_s": 8.398708109998552, + "steady_tps": 63.1, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 61.23802140639724, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 8.411343838000903, + "end_to_end_tps": 60.870178399660496, + "timing_valid": true, + "steady_tps": 63.25055323953672, + "per_stream_tps": [ + 30.6, + 31.9 + ], + "ttft_s": [ + 0.08, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08260039200104075, + "decode_tps": 30.61901070319862, + "prefill_tps": null, + "itl_mean_ms": 32.65944839607555, + "itl_p50_ms": 0.0, + "itl_p95_ms": 88.12916299939388, + "round_gap_p50_ms": 74.13766899844632, + "round_gap_p95_ms": 91.88182900106767, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1928079429999343, + "decode_tps": 31.928207987435012, + "prefill_tps": null, + "itl_mean_ms": 31.320267031382993, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.57554399906076, + "round_gap_p50_ms": 74.81583600019803, + "round_gap_p95_ms": 91.87558800113038, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 31.98985771372927, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.7446900010691, + "round_gap_p50_ms": 74.61033900108305, + "round_gap_p95_ms": 91.88182900106767, + "round_gap_mean_ms": 75.53160849074966, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 29805.84428043, + "first": 29805.926880822, + "last": 29814.255040163, + "end": 29814.255522419, + "elapsed_s": 8.4112419890007, + "chunk_offsets_s": [ + 0.08260039200104075, + 0.2601052999998501, + 0.34263597300014226, + 0.434475124002347, + 0.5204034620001039, + 0.5925056630003382, + 0.6549294169999484, + 0.7164592410008481, + 0.7855722640015301, + 0.8594568010012154, + 0.9372596329994849, + 1.0012941120003234, + 1.0763633040005516, + 1.1493395249999594, + 1.2204539439990185, + 1.2863138579996303, + 1.3435288239998044, + 1.401643201999832, + 1.4589526039999328, + 1.5208885460015154, + 1.5833194340011687, + 1.648490261999541, + 1.722049143001641, + 1.7910926990007283, + 1.8577868040010799, + 1.9311559320012748, + 2.009089445000427, + 2.079688507001265, + 2.1715703360023326, + 2.2602219520013023, + 2.3372335900021426, + 2.4056005679994996, + 2.4698325829995156, + 2.5459873210020305, + 2.638064745002339, + 2.728694864999852, + 2.8076069180024206, + 2.8845346390007762, + 2.967272842000966, + 3.0362537280016113, + 3.1052313649997814, + 3.1893240220015286, + 3.2774531850009225, + 3.368925376998959, + 3.454598611999245, + 3.5315089440009615, + 3.604962058001547, + 3.6892620590006118, + 3.7750574560013774, + 3.8618021460024465, + 3.9458408919999783, + 4.03188319200126, + 4.116358233000938, + 4.201444492002338, + 4.285813844002405, + 4.369605194999167, + 4.4452201099993545, + 4.527519023002242, + 4.601656692000688, + 4.663446647999081, + 4.725739286001044, + 4.805277874998865, + 4.872008332000405, + 4.936551135000627, + 5.021400053999969, + 5.107147621001786, + 5.170080979001796, + 5.2389714529999765, + 5.320650453999406, + 5.409478231998946, + 5.484222712999326, + 5.550178924000647, + 5.61785918500027, + 5.686450175002392, + 5.761714879001374, + 5.847503843000595, + 5.943464078001853, + 6.049643859001662, + 6.1566126520010584, + 6.241067769002257, + 6.308531254002446, + 6.370094820998929, + 6.441341731999273, + 6.515888748999714, + 6.585878424000839, + 6.665553733000706, + 6.730380570999841, + 6.790323371998966, + 6.859316984999168, + 6.936029782998958, + 7.0070784439994895, + 7.075098204000824, + 7.14016042500225, + 7.205409236001287, + 7.266579093000473, + 7.328479255000275, + 7.395950915000867, + 7.476712861000124, + 7.556108758999471, + 7.631693629999063, + 7.710672966000857, + 7.7934603659996355, + 7.871835556001315, + 7.937942208001914, + 8.008449438999378, + 8.08994501900088, + 8.179102788999444, + 8.244393600001786, + 8.310588157000893, + 8.357327067002188, + 8.410759733000305 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 74.34234299944364, + "p95": 91.88182900106767, + "max": 177.50490799880936 + } + }, + { + "start": 29805.844178581, + "first": 29806.036986524, + "last": 29814.023654617, + "end": 29814.023828272, + "elapsed_s": 8.179649690999213, + "chunk_offsets_s": [ + 0.1928079429999343, + 0.26050089899945306, + 0.34293831900140503, + 0.4348139070025354, + 0.5209971889998997, + 0.5930989740008954, + 0.6553804390023288, + 0.716816034000658, + 0.7853991420015518, + 0.8600094810026349, + 0.937751045999903, + 1.0019549429998733, + 1.0767707790000713, + 1.1496984450022865, + 1.2208357570016233, + 1.2868001770002593, + 1.3439947789993312, + 1.4021988390013576, + 1.459389359002671, + 1.5214191289996961, + 1.5833362440025667, + 1.6489264880001429, + 1.7224002340008155, + 1.791456693001237, + 1.8582499580006697, + 1.931674028000998, + 2.0094115110005077, + 2.0792870889999904, + 2.1717625070014037, + 2.2604964390011446, + 2.337058776000049, + 2.4057764900026086, + 2.4702474510013417, + 2.5462969950021943, + 2.638319509002031, + 2.7292329049996624, + 2.808022555000207, + 2.884780087002582, + 2.9678017040023406, + 3.0365442059992347, + 3.1058473299999605, + 3.189670866999222, + 3.277795611000329, + 3.3693086250023043, + 3.455006881002191, + 3.531347915999504, + 3.6054004399993573, + 3.6895809689995076, + 3.775503426000796, + 3.862078969999857, + 3.9456716690001485, + 4.031677598999522, + 4.11659801100177, + 4.201342937001755, + 4.286104961000092, + 4.369329948000086, + 4.445031148999988, + 4.527703219002433, + 4.601946769002097, + 4.663982631002, + 4.725998866000737, + 4.80544989000191, + 4.87181053400127, + 4.936915746999148, + 5.021729045001848, + 5.1070300430001225, + 5.170264341999427, + 5.239218866001465, + 5.320977028000925, + 5.409842810000555, + 5.48485330100084, + 5.550789438002539, + 5.618107030000829, + 5.686677945999691, + 5.7623843510009465, + 5.84747554400019, + 5.944167179000942, + 6.050250270000106, + 6.156814199999644, + 6.241255341999931, + 6.308872200999758, + 6.370409065999411, + 6.441581731000042, + 6.516433160999441, + 6.586625356001605, + 6.666101090002485, + 6.730959560001793, + 6.7906953589990735, + 6.859539760000189, + 6.9364469300016935, + 7.00756581600217, + 7.075407438002003, + 7.140090091001184, + 7.205735917999846, + 7.26711839500058, + 7.329045305999898, + 7.396532430000661, + 7.477086671999132, + 7.556654608000827, + 7.6316695120003715, + 7.711282527001458, + 7.793869618002645, + 7.8721588289990905, + 7.938199255000654, + 8.008913532001316, + 8.08997448000082, + 8.179476036002598 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 74.83363299979828, + "p95": 91.87558800113038, + "max": 106.56392999953823 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 61.36832002418478, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 8.39553391699883, + "end_to_end_tps": 60.98480514304513, + "timing_valid": true, + "steady_tps": 63.14507341911167, + "per_stream_tps": [ + 30.7, + 31.9 + ], + "ttft_s": [ + 0.08, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08474764300262905, + "decode_tps": 30.68416001209239, + "prefill_tps": null, + "itl_mean_ms": 32.59010510979957, + "itl_p50_ms": 0.0, + "itl_p95_ms": 89.03663399905781, + "round_gap_p50_ms": 75.25072300268221, + "round_gap_p95_ms": 93.18022899969947, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1663397829979658, + "decode_tps": 31.87496289355806, + "prefill_tps": null, + "itl_mean_ms": 31.372585541177223, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.31874799923389, + "round_gap_p50_ms": 75.41397000022698, + "round_gap_p95_ms": 92.19106400269084, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 31.981345325488398, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.32699200118077, + "round_gap_p50_ms": 75.41397000022698, + "round_gap_p95_ms": 93.18022899969947, + "round_gap_mean_ms": 75.51150979629205, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 29814.256887801, + "first": 29814.341635444, + "last": 29822.652112247, + "end": 29822.652307565, + "elapsed_s": 8.395419764001417, + "chunk_offsets_s": [ + 0.08474764300262905, + 0.2341477810005017, + 0.31626064400188625, + 0.4075589170024614, + 0.4937105050012178, + 0.5667370720002509, + 0.6308318539995525, + 0.6934034270016127, + 0.7670472330028133, + 0.8436460209995857, + 0.9220167150015186, + 0.9863175020000199, + 1.0602757660017232, + 1.129978059001587, + 1.1978820809999888, + 1.2638033729999734, + 1.3226411989999178, + 1.3812564200015913, + 1.439156524000282, + 1.499047489000077, + 1.5605633030027093, + 1.6248867419999442, + 1.7001374650026264, + 1.7763264810018882, + 1.8454003219994775, + 1.9174127000005683, + 1.9932766790007008, + 2.06177275400114, + 2.1454661870011478, + 2.2305364919993735, + 2.30731438300063, + 2.3754232480023347, + 2.440296087999741, + 2.516516447001777, + 2.6096966760014766, + 2.6998566780021065, + 2.7851759969998966, + 2.8635435159994813, + 2.9456668849998096, + 3.0155506299997796, + 3.0839298680002685, + 3.163680990001012, + 3.251007982002193, + 3.3432872150006006, + 3.4288305120026052, + 3.5046582499999204, + 3.5788839840024593, + 3.6640555859994492, + 3.75803464099954, + 3.84752741900229, + 3.93209244800164, + 4.017166868001368, + 4.10188453000228, + 4.181328905000555, + 4.263219754000602, + 4.347015287999966, + 4.4221829880007135, + 4.5037808360029885, + 4.5775388719994226, + 4.641037428002164, + 4.704918657000235, + 4.791253151001001, + 4.859625879002124, + 4.922178510001686, + 5.005955374999758, + 5.0915211620013, + 5.149656009001774, + 5.214992246001202, + 5.295923589001177, + 5.384971448002034, + 5.460849177001364, + 5.526274015002855, + 5.593949871999939, + 5.66251210900009, + 5.742717617002199, + 5.831754251001257, + 5.926833317000273, + 6.03261233000012, + 6.133619023999927, + 6.21435467700212, + 6.281634713999665, + 6.343573053, + 6.415044375000434, + 6.4914445659997, + 6.562407659002929, + 6.645073301002412, + 6.713817038002162, + 6.778843282001617, + 6.849949537001521, + 6.928340425001807, + 6.999908600002527, + 7.068071355999564, + 7.129275162002159, + 7.191337945001578, + 7.2508101890016405, + 7.312565282001742, + 7.378708170999744, + 7.458093699002347, + 7.537542691999988, + 7.6151859990022785, + 7.696370205001585, + 7.785870914001862, + 7.866400091999822, + 7.932691294001415, + 8.003724902002432, + 8.085001020001073, + 8.16614717700213, + 8.229219847002241, + 8.294338609000988, + 8.341837894000491, + 8.39522444600152 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 75.53923049999867, + "p95": 93.18022899969947, + "max": 149.40013799787266 + } + }, + { + "start": 29814.256773648, + "first": 29814.423113431, + "last": 29822.423122744, + "end": 29822.42339669, + "elapsed_s": 8.16662304199781, + "chunk_offsets_s": [ + 0.1663397829979658, + 0.23447226299686008, + 0.315892893999262, + 0.4078565849995357, + 0.4937397039975622, + 0.5672185410003294, + 0.6315019109970308, + 0.6937747619995207, + 0.7673663979985577, + 0.8440569429985771, + 0.9225544509972678, + 0.9866156169991882, + 1.0606501740003296, + 1.1304640259986627, + 1.1978609679972578, + 1.2640398900002765, + 1.3232441189975361, + 1.381548291999934, + 1.439515403999394, + 1.4993464689978282, + 1.5608307129987224, + 1.625953850998485, + 1.7004750699998112, + 1.7761412569998356, + 1.8454240639985073, + 1.9174698339993483, + 1.9928838039995753, + 2.0620535149973875, + 2.1457798439987528, + 2.2310653729982732, + 2.30754918599996, + 2.3756001029978506, + 2.4406805629987502, + 2.5164407800002664, + 2.6098973819971434, + 2.7000548709984287, + 2.785369514997001, + 2.863352884996857, + 2.9458887209984823, + 3.0150873409984342, + 3.084225114998844, + 3.1639721549981914, + 3.2512909029974253, + 3.343481967000116, + 3.4291522999992594, + 3.5048818629984453, + 3.579286960997706, + 3.6644140139978845, + 3.7582800869968196, + 3.84782396099763, + 3.93243608499688, + 4.0174658599971735, + 4.102616683998349, + 4.181302271997993, + 4.263478614997439, + 4.3472005299991, + 4.422521581996989, + 4.50439126999845, + 4.5778520129970275, + 4.641384280999773, + 4.705173162998108, + 4.79166516999976, + 4.859905771998456, + 4.922587070999725, + 5.006693831997836, + 5.091798589997779, + 5.150173548998282, + 5.215251169996918, + 5.296266007997474, + 5.3853489229986735, + 5.461024639000243, + 5.526682351999625, + 5.594317836999835, + 5.662069127996801, + 5.742907723997632, + 5.83159895299832, + 5.927315621996968, + 6.033040449998225, + 6.133887246996892, + 6.214648927998496, + 6.281951550998201, + 6.343958385998121, + 6.414927558998897, + 6.491393538999546, + 6.562893723999878, + 6.645410657998582, + 6.713703841000097, + 6.778755968000041, + 6.850423290998151, + 6.928780948997883, + 7.000271936998615, + 7.068428001999564, + 7.129682150996814, + 7.1916622089993325, + 7.251120813998568, + 7.313027719999809, + 7.378979002998676, + 7.457997724999586, + 7.537757513997349, + 7.615423887000361, + 7.696562471999641, + 7.785823522997816, + 7.8666651919993456, + 7.932862623998517, + 8.003977083997597, + 8.085202012000082, + 8.166349095998157 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 75.54007850012567, + "p95": 92.19106400269084, + "max": 105.72482800125726 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 61.31761671024085, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 8.398708109998552, + "end_to_end_tps": 60.96175665284411, + "timing_valid": true, + "steady_tps": 63.132178653125855, + "per_stream_tps": [ + 31.0, + 31.6 + ], + "ttft_s": [ + 0.16, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16051561000131187, + "decode_tps": 30.955016345416606, + "prefill_tps": null, + "itl_mean_ms": 32.30494175294035, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.96146700094687, + "round_gap_p50_ms": 74.2120539980533, + "round_gap_p95_ms": 95.3361860010773, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08101787599662202, + "decode_tps": 31.561933723364294, + "prefill_tps": null, + "itl_mean_ms": 31.683736768629352, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.31085899853497, + "round_gap_p50_ms": 75.4227279976476, + "round_gap_p95_ms": 96.73695500168833, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 31.99433926078485, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.97436500006006, + "round_gap_p50_ms": 74.48187699992559, + "round_gap_p95_ms": 96.73695500168833, + "round_gap_mean_ms": 75.54218992129756, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 29822.653235001, + "first": 29822.813750611, + "last": 29831.051510758, + "end": 29831.051852286, + "elapsed_s": 8.39861728500182, + "chunk_offsets_s": [ + 0.16051561000131187, + 0.2284045699998387, + 0.31268111700046575, + 0.4103803700018034, + 0.49734183700275025, + 0.5694048460027261, + 0.6324683820021164, + 0.693732073999854, + 0.76424658200267, + 0.8387284590025956, + 0.9165699500008486, + 0.9792908710005577, + 1.0521133809997991, + 1.1200791580013174, + 1.1859600300012971, + 1.2505382520030253, + 1.3120455150019552, + 1.3752348020025238, + 1.434501737003302, + 1.4944910320009512, + 1.5556227090019092, + 1.6186010880010144, + 1.6902467480031191, + 1.7604492720020062, + 1.826468613002362, + 1.8993182970007183, + 1.976161979000608, + 2.0465122950008663, + 2.1311476470000343, + 2.218971052003326, + 2.2956366590005928, + 2.3694780040023034, + 2.4370505470033095, + 2.5137280150011065, + 2.605389283002296, + 2.693985468002211, + 2.773401192000165, + 2.849256965000677, + 2.9319981560001906, + 3.00090173100034, + 3.0716939190024277, + 3.1502587880022475, + 3.2379104000028747, + 3.335067505002371, + 3.4249374740029452, + 3.501616733999981, + 3.575481286003196, + 3.6600625150022097, + 3.7465468990012596, + 3.833015787000477, + 3.917948381000315, + 4.002527561002353, + 4.087811294000858, + 4.166957360001106, + 4.250661314999888, + 4.339156241003366, + 4.419084720000683, + 4.501124094000261, + 4.57462306400339, + 4.6365538260033645, + 4.698686831001396, + 4.7778855670003395, + 4.84348918500109, + 4.906649259002734, + 4.989121165002871, + 5.073432125002, + 5.1312574809999205, + 5.1975918370008, + 5.2792743730024085, + 5.376074922001862, + 5.4553379340031825, + 5.521888451999985, + 5.5898261809998075, + 5.6573024400022405, + 5.733049750000646, + 5.818751284001337, + 5.914087470002414, + 6.019115292001516, + 6.119489903001522, + 6.201556862000871, + 6.26978742600113, + 6.334992440002679, + 6.409204494000733, + 6.484984177001024, + 6.555325824003376, + 6.636486143001093, + 6.70277804700163, + 6.766043745003117, + 6.836923050002952, + 6.915166791000956, + 6.986522595001588, + 7.054403497000749, + 7.116665731002286, + 7.180150040003355, + 7.244053798000095, + 7.307570345001295, + 7.381792140000471, + 7.465795436000917, + 7.544677473000775, + 7.6208500520006055, + 7.699787887002458, + 7.784092969002813, + 7.862335164001706, + 7.928554270001769, + 8.000295131001621, + 8.081415972003015, + 8.16016196000055, + 8.224284256000828, + 8.28997693600104, + 8.342301357002725, + 8.398275757001102 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 74.21692449861439, + "p95": 95.3361860010773, + "max": 105.02782199910143 + } + }, + { + "start": 29822.653144176, + "first": 29822.734162052, + "last": 29830.813514928, + "end": 29830.813675264, + "elapsed_s": 8.160531087996787, + "chunk_offsets_s": [ + 0.08101787599662202, + 0.22870839999814052, + 0.3125943599989114, + 0.4101431959970796, + 0.4970782179989328, + 0.5691051169997081, + 0.6321299470000667, + 0.6934438369971758, + 0.7640308569971239, + 0.8386614859991823, + 0.9164949099977093, + 0.9791091449988016, + 1.051916832999268, + 1.1199813729981543, + 1.185968514997512, + 1.250280574997305, + 1.31190521699682, + 1.3751155840000138, + 1.434448435997183, + 1.494482934998814, + 1.5554603689997748, + 1.6186043389971019, + 1.6905478720000247, + 1.7602973519969964, + 1.8268864909987315, + 1.8990562339968164, + 1.9761051399982534, + 2.0469689579986152, + 2.131162902998767, + 2.2188302899994596, + 2.295645225000044, + 2.3697920279992104, + 2.4368018829991342, + 2.513458189998346, + 2.6053353269999207, + 2.693960179996793, + 2.773350612998911, + 2.8492093309978372, + 2.9317813139969076, + 3.0007079059978423, + 3.071645020998403, + 3.150447398998949, + 3.237758257997484, + 3.335045097999682, + 3.4253391179991013, + 3.5015561029977107, + 3.5760259829985444, + 3.659978386996954, + 3.7469527519970143, + 3.8328329769974516, + 3.917886676998023, + 4.002436132999719, + 4.087548833998881, + 4.166718671996932, + 4.250441930998932, + 4.339127335999365, + 4.419254409996938, + 4.501455178997276, + 4.575044618999527, + 4.637029546996928, + 4.6986104029965645, + 4.7778339159995085, + 4.843512141997053, + 4.906947945997672, + 4.988437874999363, + 5.073593986999185, + 5.131298060998233, + 5.197579484000016, + 5.279302019996976, + 5.376038974998664, + 5.455212349999783, + 5.5215829699991446, + 5.589817829997628, + 5.657621967999148, + 5.733044695996796, + 5.818611757997132, + 5.914066920999176, + 6.019018438997591, + 6.119296354998369, + 6.201440977998573, + 6.269668133998493, + 6.3347658879974915, + 6.409215301999211, + 6.484891214997333, + 6.555128595999122, + 6.636342087997036, + 6.70280175699736, + 6.765981706997991, + 6.836725133998698, + 6.914674771000136, + 6.98692130499694, + 7.05416132299797, + 7.116322410998691, + 7.179793420997157, + 7.243977147998521, + 7.307551925998268, + 7.3818278389990155, + 7.4657939999997325, + 7.545011803998932, + 7.620524581998325, + 7.699715942999319, + 7.78382424099982, + 7.8623297909980465, + 7.9285311449966684, + 8.000148708997585, + 8.08127267099917, + 8.160370751997107 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 75.46775299852015, + "p95": 96.73695500168833, + "max": 147.6905240015185 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.2, + "per_stream_tps": 21.2, + "ttft_s_max": 0.34, + "request": { + "ttft_s": 0.209, + "decode_tps": 21.269, + "prefill_tps": null, + "itl_mean_ms": 47.023, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.479, + "round_gap_p50_ms": 109.195, + "round_gap_p95_ms": 129.549, + "tokens_per_chunk": 2.339 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 46.524, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.596, + "round_gap_p50_ms": 109.829, + "round_gap_p95_ms": 129.484, + "tokens_per_chunk": 2.359 + }, + "timing_valid": true, + "end_to_end_tps": 81.93134077233795, + "makespan_s": 12.498269774998334, + "steady_tps": 85.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.17526804282075, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 12.498269774998334, + "end_to_end_tps": 81.93134077233795, + "timing_valid": true, + "steady_tps": 85.54063884843183, + "per_stream_tps": [ + 21.0, + 21.5, + 20.5, + 23.0 + ], + "ttft_s": [ + 0.25, + 0.17, + 0.09, + 0.33 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.24707734500043443, + "decode_tps": 20.992168209041765, + "prefill_tps": null, + "itl_mean_ms": 47.6368134078346, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.94149299830315, + "round_gap_p50_ms": 109.17087599955266, + "round_gap_p95_ms": 129.6385419991566, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16579057700073463, + "decode_tps": 21.52767711853408, + "prefill_tps": null, + "itl_mean_ms": 46.45183010196014, + "itl_p50_ms": 0.0, + "itl_p95_ms": 122.12464600088424, + "round_gap_p50_ms": 109.95608799930778, + "round_gap_p95_ms": 129.72444600018207, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.0855848130013328, + "decode_tps": 20.543817010705187, + "prefill_tps": null, + "itl_mean_ms": 48.67644603137331, + "itl_p50_ms": 0.0, + "itl_p95_ms": 122.84868100323365, + "round_gap_p50_ms": 108.99262300154078, + "round_gap_p95_ms": 129.46030699822586, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33166613099820097, + "decode_tps": 23.018994327812475, + "prefill_tps": null, + "itl_mean_ms": 43.44238439607936, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.35552600011579, + "round_gap_p50_ms": 110.69233099988196, + "round_gap_p95_ms": 126.86714200026472, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 46.55186848431185, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.94149299830315, + "round_gap_p50_ms": 109.82938600136549, + "round_gap_p95_ms": 129.4842580027762, + "round_gap_mean_ms": 110.42536245115835, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 29831.054419355, + "first": 29831.3014967, + "last": 29843.448884119, + "end": 29843.44904488, + "elapsed_s": 12.394625524997537, + "chunk_offsets_s": [ + 0.24707734500043443, + 0.44352435099790455, + 0.5733001439984946, + 0.6997836029986502, + 0.8140879799975664, + 0.9145686809970357, + 1.0162245169995003, + 1.1262153429997852, + 1.246453046998795, + 1.349718268000288, + 1.4488290389999747, + 1.5446827529995062, + 1.6461929319993942, + 1.7516568379978708, + 1.8593080999999074, + 1.985083086998202, + 2.0964644889972988, + 2.212761483999202, + 2.3167070639974554, + 2.4153957039998204, + 2.526069726998685, + 2.62917946199741, + 2.7322531649988377, + 2.8443259969972132, + 2.9502272049976455, + 3.0641016099980334, + 3.1779216909999377, + 3.2781810950000363, + 3.3977143989977776, + 3.5190371289972973, + 3.6307573769990995, + 3.7319159589969786, + 3.8305118729986134, + 3.9362578529980965, + 4.0565026859985664, + 4.186141227997723, + 4.299161467999511, + 4.412613872998918, + 4.53014003899807, + 4.63111197499893, + 4.737709487999382, + 4.849879760997283, + 4.970469018997392, + 5.086849487997824, + 5.202458866999223, + 5.306651409999176, + 5.413268562999292, + 5.52702026499901, + 5.646626830999594, + 5.754903920998913, + 5.868103780998354, + 5.982890424998914, + 6.1078159129974665, + 6.234468782997283, + 6.375386613999581, + 6.4998441509997065, + 6.610006443999737, + 6.720838329998514, + 6.84166171600009, + 6.950832591999642, + 7.0723610969980655, + 7.191238428000361, + 7.3029175759984355, + 7.420357383998635, + 7.537515886997426, + 7.650125828000455, + 7.749550554999587, + 7.850599082998087, + 7.96522369199738, + 8.091182696000033, + 8.194343213999673, + 8.29941456099914, + 8.407492417998583, + 8.518751408999378, + 8.618620176999684, + 8.732124310998188, + 8.851638823998655, + 8.993035619998409, + 9.126999059997615, + 9.230339729998377, + 9.326034533998609, + 9.415548949997174, + 9.520384482999361, + 9.622368899999856, + 9.71775360799802, + 9.833020871999906, + 9.948510619000444, + 10.04439287599962, + 10.142140779997135, + 10.241402429997834, + 10.343388643999788, + 10.438065586997254, + 10.531638260999898, + 10.631053516997781, + 10.728364288999728, + 10.836850366998988, + 10.95879185999729, + 11.075802809999004, + 11.18163700399964, + 11.289576652998221, + 11.408481157999631, + 11.505445434999274, + 11.599005297997792, + 11.689267246998497, + 11.792558686996927, + 11.898922664997372, + 12.010284891999618, + 12.086171928996919, + 12.165725270999246, + 12.251314475997788, + 12.326439083997684, + 12.394464763998258 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 109.17087599955266, + "p95": 129.6385419991566, + "max": 196.44700599747011 + } + }, + { + "start": 29831.053857419, + "first": 29831.219647996, + "last": 29843.064864672, + "end": 29843.064965107, + "elapsed_s": 12.011107688002085, + "chunk_offsets_s": [ + 0.16579057700073463, + 0.44417510600032983, + 0.5740416549997462, + 0.7005283959988446, + 0.8149439030021313, + 0.9154088069990394, + 1.01688604499941, + 1.1267154310007754, + 1.2468502150004497, + 1.3505889020016184, + 1.4494623150021653, + 1.545373508000921, + 1.6466231039994454, + 1.7520600079988071, + 1.859517835000588, + 1.985597756000061, + 2.096761698001501, + 2.2136800859989307, + 2.317039327001112, + 2.416276068001025, + 2.5264349460012454, + 2.629922076001094, + 2.733211430000665, + 2.8451385580010538, + 2.950967208998918, + 3.0646213379986875, + 3.178088499000296, + 3.2785913560001063, + 3.3979102420016716, + 3.519457250000414, + 3.6309342050008127, + 3.7321578199989744, + 3.830844377000176, + 3.93690337299995, + 4.056686029998673, + 4.186410475998855, + 4.29946228100016, + 4.413022806998924, + 4.530520289001288, + 4.631434588001866, + 4.737927967002179, + 4.8510972570002195, + 4.970569517001422, + 5.087547907998669, + 5.2023800810020475, + 5.307293583999126, + 5.413558348998777, + 5.527267553999991, + 5.646769489001599, + 5.755516532000911, + 5.86804772600226, + 5.983635950000462, + 6.108689341999707, + 6.2351101560016104, + 6.376179228002002, + 6.500347697001416, + 6.610303785000724, + 6.7208876270015026, + 6.841663717001211, + 6.951232788000198, + 7.072441158001311, + 7.191495628001576, + 7.302983904999564, + 7.421160177000274, + 7.53736096600187, + 7.65002452599947, + 7.7498525560004055, + 7.850825040000927, + 7.965478426998743, + 8.091165891000855, + 8.195391471999756, + 8.300411600001098, + 8.408005024000886, + 8.519593815999542, + 8.618727945999126, + 8.732345338001323, + 8.852267632999428, + 8.993730403999507, + 9.126443373999791, + 9.230421027001285, + 9.32594450599936, + 9.416211162999389, + 9.52113928900144, + 9.622800869001367, + 9.717937247001828, + 9.833919472999696, + 9.948586059999798, + 10.044315782000922, + 10.142446879999625, + 10.241172353002185, + 10.343799116999435, + 10.438053284000489, + 10.53166113900079, + 10.631911656000739, + 10.728651453999191, + 10.836888196001382, + 10.959012842002267, + 11.07593915400139, + 11.18197263500042, + 11.29045835099896, + 11.409365641000477, + 11.50562430699938, + 11.59875724900121, + 11.689592203001666, + 11.792820196998946, + 11.899295014001837, + 12.011007253000571 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 110.05748299976403, + "p95": 129.72444600018207, + "max": 278.3845289995952 + } + }, + { + "start": 29831.053747616, + "first": 29831.139332429, + "last": 29843.551826167, + "end": 29843.55191988, + "elapsed_s": 12.498172264000459, + "chunk_offsets_s": [ + 0.0855848130013328, + 0.4443950330023654, + 0.5738553400005912, + 0.700308996001695, + 0.8143834270013031, + 0.9157309170004737, + 1.0168225950001215, + 1.1264967530005379, + 1.246703549000813, + 1.3501225899999554, + 1.4491458869997587, + 1.5450959270019666, + 1.6470813480009383, + 1.7517018370017468, + 1.85975866499939, + 1.985595596001076, + 2.0971962030016584, + 2.213350630001514, + 2.316922706002515, + 2.4158754640011466, + 2.526077558999532, + 2.6296244410004874, + 2.7332604410003114, + 2.844859570999688, + 2.950656914999854, + 3.064573769002891, + 3.178758934001962, + 3.2786310680021415, + 3.3987966640015657, + 3.5199349070026074, + 3.630904945999646, + 3.7321005500016327, + 3.8314415340028063, + 3.9372431030024018, + 4.056849661999877, + 4.186333920002653, + 4.300029586000164, + 4.413021334999939, + 4.530543675002264, + 4.631842617000075, + 4.738437792999321, + 4.850376009999309, + 4.970848247001413, + 5.087708338000084, + 5.202812228999392, + 5.30717149100019, + 5.413510236001457, + 5.527202699002373, + 5.646801775001222, + 5.755794398002763, + 5.868633979000151, + 5.983625051001582, + 6.108748197999375, + 6.234977677002462, + 6.375933666000492, + 6.50059051000062, + 6.610305467002036, + 6.720858545999363, + 6.8420383699995, + 6.951100678001239, + 7.073268427000585, + 7.191515061000246, + 7.303950793000695, + 7.420911479002825, + 7.538006886999938, + 7.650577325002814, + 7.749747304002085, + 7.851004693002324, + 7.965431485001318, + 8.091620093000529, + 8.19509322200247, + 8.300603081002919, + 8.408467836001364, + 8.519542919999367, + 8.61912529500114, + 8.733050590999483, + 8.851781011002458, + 8.9941947360021, + 9.127273663001688, + 9.231233037000493, + 9.326947961002588, + 9.416081423001742, + 9.521097581000504, + 9.623104278001847, + 9.717826105999848, + 9.833472260001145, + 9.949123520000285, + 10.044688639001834, + 10.142244859001948, + 10.24186266900142, + 10.344151503999456, + 10.438621780001995, + 10.53169841899944, + 10.631584742000996, + 10.728608178000286, + 10.836532872999669, + 10.959381554002903, + 11.075870429001952, + 11.18194605300232, + 11.29011703300057, + 11.409259943000507, + 11.505461788001412, + 11.59994668500076, + 11.68962878000093, + 11.793042279001384, + 11.899311872002727, + 12.011051479999878, + 12.086656769999536, + 12.166055993002374, + 12.252266234001581, + 12.326906141999643, + 12.395899140999973, + 12.447415573002218, + 12.498078551001527 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 108.99262300154078, + "p95": 129.46030699822586, + "max": 358.8102200010326 + } + }, + { + "start": 29831.053650105, + "first": 29831.385316236, + "last": 29842.463124257, + "end": 29842.46367559, + "elapsed_s": 11.410025484998187, + "chunk_offsets_s": [ + 0.33166613099820097, + 0.44420831900060875, + 0.5741995089992997, + 0.7004855470004259, + 0.8147516040007758, + 0.9154341129979002, + 1.017046107997885, + 1.1267898929982039, + 1.2471345029989607, + 1.350680249997822, + 1.4494605389991193, + 1.5456443990005937, + 1.6466963489983755, + 1.75191425599769, + 1.8596493119985098, + 1.985915016997751, + 2.097489214000234, + 2.2138325790001545, + 2.317192859998613, + 2.416646468998806, + 2.5265326169974287, + 2.630196359998081, + 2.7332668290000584, + 2.8454628929976025, + 2.951230241000303, + 3.064764947997901, + 3.178466278997803, + 3.278842366999015, + 3.398269838999113, + 3.5196253649992286, + 3.631207192000147, + 3.7325163280002016, + 3.832213382000191, + 3.9368856729997788, + 4.057015982998564, + 4.186699228997895, + 4.299577302997932, + 4.413272424997558, + 4.530487239997456, + 4.631703061000735, + 4.738180577998719, + 4.850956214999314, + 4.970835028998408, + 5.087861709998833, + 5.202181802000268, + 5.307553189999453, + 5.41382542899737, + 5.527528311999049, + 5.647041883999918, + 5.7555589209987374, + 5.868477091000386, + 5.983899940998526, + 6.108656981999957, + 6.2355241240002215, + 6.376316050998867, + 6.5002519539993955, + 6.610555691000627, + 6.721248022000509, + 6.841809135999938, + 6.951494778997585, + 7.072777639998094, + 7.191474633000325, + 7.303430939999089, + 7.4215987129973655, + 7.537988788000803, + 7.650480363998213, + 7.750546175997442, + 7.851600530000724, + 7.965840501998173, + 8.092037368998717, + 8.195347254000808, + 8.300756660999468, + 8.408408729999792, + 8.519869570998708, + 8.61889218899887, + 8.732280312000512, + 8.852124899000046, + 8.994472185000632, + 9.127519948000554, + 9.23090570599743, + 9.326473103999888, + 9.416656867000711, + 9.521669072000805, + 9.622910466998292, + 9.71820206999837, + 9.83425678599815, + 9.948884061999706, + 10.044869539997308, + 10.142776942000637, + 10.241643889999978, + 10.344054704997689, + 10.438218390998372, + 10.53221528100039, + 10.631906541999342, + 10.728908881999814, + 10.836838265997358, + 10.959345336999831, + 11.076217164998525, + 11.182228061999922, + 11.290895842997998, + 11.409474151998438 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 111.07658599939896, + "p95": 126.86714200026472, + "max": 142.34728600058588 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.10689430820052, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 12.508602719000919, + "end_to_end_tps": 81.86365999493414, + "timing_valid": true, + "steady_tps": 85.67918161326931, + "per_stream_tps": [ + 20.7, + 21.7, + 20.9, + 22.7 + ], + "ttft_s": [ + 0.09, + 0.25, + 0.34, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08560655499968561, + "decode_tps": 20.72120377593873, + "prefill_tps": null, + "itl_mean_ms": 48.25974450196715, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.62919799811789, + "round_gap_p50_ms": 108.60249799952726, + "round_gap_p95_ms": 129.82939100038493, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2510599409979477, + "decode_tps": 21.710224120891947, + "prefill_tps": null, + "itl_mean_ms": 46.0612472000089, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.61208099860232, + "round_gap_p50_ms": 109.21892699843738, + "round_gap_p95_ms": 130.45125699864002, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3352698170019721, + "decode_tps": 20.94859015903998, + "prefill_tps": null, + "itl_mean_ms": 47.73590930979516, + "itl_p50_ms": 0.0, + "itl_p95_ms": 120.97844299933058, + "round_gap_p50_ms": 108.54174999985844, + "round_gap_p95_ms": 129.74603200200363, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17019689500011737, + "decode_tps": 22.707467690862963, + "prefill_tps": null, + "itl_mean_ms": 44.038375992157874, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.60186099936254, + "round_gap_p50_ms": 109.96671300017624, + "round_gap_p95_ms": 129.9571140007174, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 46.52381925098227, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.5960269983043, + "round_gap_p50_ms": 109.27060800167965, + "round_gap_p95_ms": 129.82939100038493, + "round_gap_mean_ms": 110.35882706046958, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 29843.553375147, + "first": 29843.638981702, + "last": 29855.94521655, + "end": 29855.945332107, + "elapsed_s": 12.391956960000243, + "chunk_offsets_s": [ + 0.08560655499968561, + 0.45261123700038297, + 0.5878273740017903, + 0.7135914430000412, + 0.8248736810019182, + 0.9237552960003086, + 1.0178889090020675, + 1.1264914070015948, + 1.2469404350013065, + 1.3522163429988723, + 1.4545528929993452, + 1.553612735002389, + 1.6552404790018045, + 1.7598786140006268, + 1.8684127279993845, + 1.988283267000952, + 2.096763155001099, + 2.2139447260015004, + 2.314331958001276, + 2.4155006210021384, + 2.5334535729998606, + 2.6389728260000993, + 2.7423231540014967, + 2.8527483719990414, + 2.952838391000114, + 3.062762252000539, + 3.1757350209991273, + 3.277004528001271, + 3.3973844739994092, + 3.527213864999794, + 3.6392151020008896, + 3.740522308002255, + 3.838648714001465, + 3.941046054002072, + 4.057614871002443, + 4.186519817001681, + 4.299184182000317, + 4.414897429000121, + 4.539658470999711, + 4.639533534998918, + 4.747270104002382, + 4.856184224001481, + 4.969256029002281, + 5.086463512001501, + 5.199622514999646, + 5.305587114999071, + 5.409754495001835, + 5.530447193999862, + 5.651559026999166, + 5.76154396900165, + 5.872961250999651, + 5.983023436001531, + 6.1039805760010495, + 6.228979335999611, + 6.369815198999277, + 6.502720956999838, + 6.614778309001849, + 6.724950277999596, + 6.844354714001383, + 6.9494860189988685, + 7.0657801820016175, + 7.184542659000726, + 7.296520587999112, + 7.414051431002008, + 7.53908923400013, + 7.651088782000443, + 7.749546401999396, + 7.851473896000243, + 7.960863632000837, + 8.082492829998955, + 8.18489000000045, + 8.290230548002, + 8.400457845000346, + 8.519066038999881, + 8.618761042002006, + 8.730476094999176, + 8.850423465999484, + 8.986655271000927, + 9.116358169001614, + 9.22077079399969, + 9.315030378998927, + 9.407041231999756, + 9.520316885002103, + 9.62189601299906, + 9.718047488000593, + 9.831540028000745, + 9.942280465002113, + 10.036221359001502, + 10.133740019002289, + 10.232905539000058, + 10.336780397999974, + 10.43597339400003, + 10.53204117900168, + 10.632859600998927, + 10.729762202001439, + 10.835932512, + 10.952278297001612, + 11.064606980999088, + 11.172071984001377, + 11.27861829000176, + 11.399784530000034, + 11.502546890998929, + 11.599821730000258, + 11.689512626999203, + 11.79323056100111, + 11.898836497999582, + 11.996620687001268, + 12.073347738001758, + 12.15611760899992, + 12.244272209001792, + 12.318986473001132, + 12.391841403001308 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 108.60249799952726, + "p95": 129.82939100038493, + "max": 367.00468200069736 + } + }, + { + "start": 29843.553663505, + "first": 29843.804723446, + "last": 29855.550341482, + "end": 29855.550472996, + "elapsed_s": 11.996809491000022, + "chunk_offsets_s": [ + 0.2510599409979477, + 0.452649440998357, + 0.5871262469991052, + 0.7136465499970654, + 0.8253939899987017, + 0.9228266649988655, + 1.0174647120002192, + 1.12615623199963, + 1.2464429479987302, + 1.3514423279993935, + 1.4539473049990193, + 1.5536861189975752, + 1.655236619997595, + 1.7596629209983803, + 1.8686071939991962, + 1.9880437100000563, + 2.09666761599874, + 2.2135022500006016, + 2.314079691997904, + 2.4150591539982997, + 2.533043572999304, + 2.638449378999212, + 2.7417346599977463, + 2.8525693640003738, + 2.9526427529999637, + 3.062225039997429, + 3.1760862599985558, + 3.2768223179991764, + 3.397809767997387, + 3.527169334000064, + 3.6391204100000323, + 3.7394944169973314, + 3.8381416729971534, + 3.940698283000529, + 4.057293188998301, + 4.186475446997065, + 4.299383765999664, + 4.4145682099988335, + 4.539625655997952, + 4.639522743997077, + 4.7476230669999495, + 4.8560764549984015, + 4.969027066999843, + 5.086111387998244, + 5.199406340998394, + 5.305388370998116, + 5.409806395997293, + 5.530268921000243, + 5.651881001998845, + 5.761783196998294, + 5.8723186779971, + 5.982479791997321, + 6.103938364998612, + 6.228388891999202, + 6.369693952998205, + 6.502656897999259, + 6.614726941999834, + 6.724736055999529, + 6.844546181000624, + 6.948695348997717, + 7.06557673199859, + 7.184904646997893, + 7.296121584997309, + 7.4140065779974975, + 7.539142669997091, + 7.650560071997461, + 7.749199704998318, + 7.851227631999791, + 7.960446558998228, + 8.081862741997611, + 8.184719826000219, + 8.289986877000047, + 8.400890316999721, + 8.518895799999882, + 8.61820720399919, + 8.73073767799724, + 8.850277794997965, + 8.986072960000456, + 9.116524216999096, + 9.220953855998232, + 9.314959916999214, + 9.407049131998065, + 9.519769642000028, + 9.621878111000115, + 9.717527117998543, + 9.830494366000494, + 9.94253793399912, + 10.035797148997517, + 10.133281909998914, + 10.232488227000431, + 10.336873092997848, + 10.435855315998197, + 10.53259605600033, + 10.632637216000148, + 10.729531526998471, + 10.83606406599938, + 10.952303272999416, + 11.064246310997987, + 11.171853600997565, + 11.279001269998844, + 11.399806607998471, + 11.502348305999476, + 11.599799842999346, + 11.689449861998582, + 11.79302048399768, + 11.89879610799835, + 11.996677977000218 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 109.4006069979514, + "p95": 130.45125699864002, + "max": 201.58950000040932 + } + }, + { + "start": 29843.553885135, + "first": 29843.889154952, + "last": 29856.061811826, + "end": 29856.061882376, + "elapsed_s": 12.507997241002158, + "chunk_offsets_s": [ + 0.3352698170019721, + 0.45235741900251014, + 0.5876091360005375, + 0.7128126570023596, + 0.8242154170002323, + 0.9231620150021627, + 1.01707329400233, + 1.1256979429999774, + 1.2459006780009076, + 1.3511441760019807, + 1.4536491520011623, + 1.5533768909990613, + 1.654621349000081, + 1.7596205369991367, + 1.8681622869989951, + 1.9874706299997342, + 2.097218718001386, + 2.2126950240017322, + 2.3135101500010933, + 2.415090571001201, + 2.533046885000658, + 2.638016651999351, + 2.7413112809990707, + 2.8519254279999586, + 2.9520719460015243, + 3.0617314760020236, + 3.1756816070010245, + 3.277241371000855, + 3.3973789629999374, + 3.527147117001732, + 3.6390026560002298, + 3.7395618459995603, + 3.8382340220014157, + 3.940949294999882, + 4.056491905001167, + 4.185655820001557, + 4.298843404001673, + 4.414138445001299, + 4.539760466999724, + 4.639607106000767, + 4.747151881001628, + 4.855571193002106, + 4.969037404000119, + 5.086535418002313, + 5.1997331860002305, + 5.30494693899891, + 5.410069937999651, + 5.530276025001513, + 5.650958752001316, + 5.761468992001028, + 5.872223698999733, + 5.9828113410003425, + 6.103789783999673, + 6.228229250998993, + 6.3696939010005735, + 6.5019191749997844, + 6.61483873600082, + 6.7239869219993125, + 6.844827935001376, + 6.94904997499907, + 7.065633322999929, + 7.184431764999317, + 7.295567909000965, + 7.413369337999029, + 7.539161904001958, + 7.6507319010015635, + 7.7495319549998385, + 7.850736921001953, + 7.959869338999852, + 8.081832984000357, + 8.18390572700082, + 8.289463242999773, + 8.400306168001407, + 8.519109433000267, + 8.618561444000079, + 8.730311789000552, + 8.849668548999034, + 8.986314026999025, + 9.116060059001029, + 9.21996935000061, + 9.314997282999684, + 9.406635630999517, + 9.519448122999165, + 9.621844015000534, + 9.716936438999255, + 9.831259907001368, + 9.942016478002188, + 10.03503670200007, + 10.133620976001112, + 10.232503810999333, + 10.33606888900249, + 10.43570586900023, + 10.532069414999569, + 10.632506064001063, + 10.729565900001035, + 10.835552309999912, + 10.95226389399977, + 11.064520043000812, + 11.171793479999906, + 11.278691083000012, + 11.399147781001375, + 11.502308046001417, + 11.599643690002267, + 11.689520358002483, + 11.793026496001403, + 11.898663850999583, + 11.996224639999127, + 12.072585572001117, + 12.155387726001209, + 12.243491348999669, + 12.31862440400073, + 12.391211440000916, + 12.450644307002221, + 12.507926690999739 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 108.54174999985844, + "p95": 129.74603200200363, + "max": 141.4646500015806 + } + }, + { + "start": 29843.553279657, + "first": 29843.723476552, + "last": 29854.95326243, + "end": 29854.953507902, + "elapsed_s": 11.400228244998289, + "chunk_offsets_s": [ + 0.17019689500011737, + 0.45277534300112166, + 0.5878676610009279, + 0.7135221079988696, + 0.8259871439986455, + 0.9232959990004019, + 1.0173688389986637, + 1.1268378969980404, + 1.24678927900095, + 1.352062482998008, + 1.454787247999775, + 1.5544007380012772, + 1.6555512599989015, + 1.7603305480006384, + 1.8689592230002745, + 1.988744467998913, + 2.0976220779994037, + 2.2136371470005543, + 2.314509748001001, + 2.415506927998649, + 2.533862162999867, + 2.6392682410005364, + 2.741849087000446, + 2.852468500001123, + 2.952613929999643, + 3.062389531998633, + 3.1760723869992944, + 3.277509021998412, + 3.397713179998391, + 3.5276702939991083, + 3.6396783019990835, + 3.7403935939983057, + 3.8386388399994757, + 3.9414358520007227, + 4.057433353998931, + 4.18634811100128, + 4.299332457998389, + 4.414897415001178, + 4.540245567997772, + 4.640325037998991, + 4.747278587998153, + 4.8566555850011355, + 4.969695812000282, + 5.086800651999511, + 5.200529704998189, + 5.306122209000023, + 5.410411838998698, + 5.530265679000877, + 5.651861705999181, + 5.761828418999357, + 5.8728967669994745, + 5.983281028999045, + 6.104666905001068, + 6.228519903001143, + 6.370384415000444, + 6.503198606998922, + 6.61517908499809, + 6.724699107999186, + 6.844704000999627, + 6.9493035599989526, + 7.066001602001052, + 7.18477462699957, + 7.29630512699805, + 7.414211309998791, + 7.539834267998231, + 7.651020856999821, + 7.74984285499886, + 7.851441727998463, + 7.960712336000142, + 8.082314196999505, + 8.1846500190004, + 8.290197130998422, + 8.401125861000764, + 8.519476320998365, + 8.618918616000883, + 8.731038587000512, + 8.850427053999738, + 8.986827507000271, + 9.116339125001105, + 9.221118494999246, + 9.315543365999474, + 9.406944370999554, + 9.52069033799853, + 9.622215272000176, + 9.717300125998008, + 9.831079767998745, + 9.942462046998116, + 10.035969182998087, + 10.134626217000914, + 10.232962121001037, + 10.337005370998668, + 10.43638798000029, + 10.532550355997955, + 10.6336428350005, + 10.730393498997728, + 10.836797579999256, + 10.952625341000385, + 11.065044245999161, + 11.172116785997787, + 11.279094768000505, + 11.399982773000374 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 110.17548749987327, + "p95": 129.9571140007174, + "max": 282.5784480010043 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.23793658218108, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 12.490258428999368, + "end_to_end_tps": 81.98389215250494, + "timing_valid": true, + "steady_tps": 85.58886901603269, + "per_stream_tps": [ + 21.0, + 21.6, + 21.0, + 22.6 + ], + "ttft_s": [ + 0.25, + 0.17, + 0.33, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.249437874997966, + "decode_tps": 21.01112825956672, + "prefill_tps": null, + "itl_mean_ms": 47.59382683529539, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.30051599888247, + "round_gap_p50_ms": 108.73771900151041, + "round_gap_p95_ms": 127.82737000088673, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1677486640001007, + "decode_tps": 21.559336050366046, + "prefill_tps": null, + "itl_mean_ms": 46.383617643133384, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.235598999192, + "round_gap_p50_ms": 109.7964079999656, + "round_gap_p95_ms": 127.91637400005129, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33469564500046545, + "decode_tps": 20.979688978874734, + "prefill_tps": null, + "itl_mean_ms": 47.665148945103, + "itl_p50_ms": 0.0, + "itl_p95_ms": 120.6779469976027, + "round_gap_p50_ms": 108.44252400056575, + "round_gap_p95_ms": 127.43669200062868, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.0868811549989914, + "decode_tps": 22.55639218656932, + "prefill_tps": null, + "itl_mean_ms": 44.33333095686405, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.19806799819344, + "round_gap_p50_ms": 110.66247800044948, + "round_gap_p95_ms": 127.97940299788024, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 46.49398109509895, + "itl_p50_ms": 0.0, + "itl_p95_ms": 121.03880799986655, + "round_gap_p50_ms": 109.85612199874595, + "round_gap_p95_ms": 127.91637400005129, + "round_gap_mean_ms": 110.28804817907194, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 29856.063548529, + "first": 29856.312986404, + "last": 29868.449412247, + "end": 29868.449598639, + "elapsed_s": 12.386050109998905, + "chunk_offsets_s": [ + 0.249437874997966, + 0.4495708789982018, + 0.5783006989986461, + 0.7061280689995328, + 0.8189589209978294, + 0.9197695449984167, + 1.0196652199992968, + 1.1284029390008072, + 1.2480397279978206, + 1.3513048960012384, + 1.4505033249988628, + 1.5470240809991083, + 1.6480274540008395, + 1.7537866269994993, + 1.8612965649990656, + 1.9887314459992922, + 2.1008843009985867, + 2.216042350999487, + 2.3162870679989283, + 2.416271477999544, + 2.527114027998323, + 2.631808910999098, + 2.735254333998455, + 2.8468518949994177, + 2.9512267419995624, + 3.064960241998051, + 3.1755429539989564, + 3.2758562929993786, + 3.3950051210013044, + 3.516305637000187, + 3.628971744001319, + 3.728959207001026, + 3.8276831959992705, + 3.935550089998287, + 4.0557175379981345, + 4.183071485000255, + 4.295993359999557, + 4.4083813069992175, + 4.524449962998915, + 4.623783006998565, + 4.730632208000316, + 4.841440555999725, + 4.958977791000507, + 5.077393155999744, + 5.1924395649984945, + 5.296827776997816, + 5.402439076999144, + 5.517243553000299, + 5.6358551400007855, + 5.745887558001414, + 5.8572510609992605, + 5.975829287999659, + 6.100562547999289, + 6.2258918970001105, + 6.367028992997803, + 6.493723839001177, + 6.603579960999923, + 6.715930283000489, + 6.835261841999454, + 6.943107021001197, + 7.062932864999311, + 7.1829552659983165, + 7.2934764249985164, + 7.411858911000309, + 7.528562494000653, + 7.638516357001208, + 7.7385094839992234, + 7.840285170001152, + 7.956637949999276, + 8.080024126000353, + 8.183555152998451, + 8.289056466001057, + 8.396865833001357, + 8.507944910001243, + 8.605259158997796, + 8.717530539001018, + 8.836836293998203, + 8.976219550000678, + 9.11027723799998, + 9.214110919998348, + 9.308794040000066, + 9.401946660000249, + 9.50766400700013, + 9.608823909999046, + 9.703027458999713, + 9.816450690999773, + 9.933579791999364, + 10.030440268998063, + 10.129331828000431, + 10.227524501999142, + 10.329193916000804, + 10.423815310001373, + 10.516397363000578, + 10.617297821998363, + 10.714643567000167, + 10.82126358199821, + 10.941723349998938, + 11.059318504998373, + 11.166289365999546, + 11.272879855998326, + 11.391407166000135, + 11.487009954998939, + 11.581382850999944, + 11.67249787900073, + 11.776196619000984, + 11.883876714000507, + 11.995417158999771, + 12.071274367001024, + 12.152314963001118, + 12.240300440000283, + 12.314695797998866, + 12.38586371799829 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 108.73771900151041, + "p95": 127.82737000088673, + "max": 200.13300400023581 + } + }, + { + "start": 29856.06331202, + "first": 29856.231060684, + "last": 29868.058883183, + "end": 29868.059131098, + "elapsed_s": 11.99581907800166, + "chunk_offsets_s": [ + 0.1677486640001007, + 0.4494113879991346, + 0.578135398998711, + 0.7060517729987623, + 0.8191209899996466, + 0.9201751639993745, + 1.0196830829991086, + 1.1294763080004486, + 1.2479526450006233, + 1.3515942220001307, + 1.4508471160006593, + 1.547096664999117, + 1.649031519998971, + 1.7542893960016954, + 1.8617983430012828, + 1.9889311279985122, + 2.1009265719985706, + 2.2160914560008678, + 2.316925271999935, + 2.4162028020000434, + 2.5268941200010886, + 2.631203089000337, + 2.7351977750004153, + 2.8471831700007897, + 2.951414579001721, + 3.0648296590006794, + 3.1760559829999693, + 3.276367497001047, + 3.3955019040004117, + 3.5167375029996037, + 3.6292741290017148, + 3.7295600549987284, + 3.827617481998459, + 3.9358821169989824, + 4.0557263439986855, + 4.183363387001009, + 4.296009254998353, + 4.408659477001493, + 4.52478533500107, + 4.624174571999902, + 4.730915084001026, + 4.84128223800144, + 4.9591307699993195, + 5.07755508199989, + 5.192873782998504, + 5.297341621000669, + 5.402880737998203, + 5.517397397001332, + 5.636370360000001, + 5.746166767999966, + 5.85740458500004, + 5.9762382180015265, + 6.100851472998329, + 6.225782569999865, + 6.366888152999309, + 6.492820808998658, + 6.603888221001398, + 6.715806487998634, + 6.835595356998965, + 6.943171507999068, + 7.063325949999125, + 7.183118568998907, + 7.29357710000113, + 7.412396217001515, + 7.528928898998856, + 7.638379741998506, + 7.739253109000856, + 7.840754406999622, + 7.956806949998281, + 8.08042590200057, + 8.184000942001148, + 8.289469701001508, + 8.397025342001143, + 8.508084076998784, + 8.605377648000285, + 8.717947103999904, + 8.83692787900145, + 8.976410012000997, + 9.110448239000107, + 9.214432846998534, + 9.308724615999381, + 9.40194341499955, + 9.507550199999969, + 9.60922342900085, + 9.70306328299921, + 9.816758982000465, + 9.933859257998847, + 10.030874405998475, + 10.129486951998842, + 10.227883209001448, + 10.329356275000464, + 10.424188388999028, + 10.516779916000814, + 10.617472248999547, + 10.714827452000463, + 10.820894483000302, + 10.941873784999189, + 11.05940562500109, + 11.166424357001233, + 11.273032419998344, + 11.39158940299967, + 11.4870214520015, + 11.581454478000524, + 11.67264184799933, + 11.776381431998743, + 11.884051252000063, + 11.995571162999113 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 110.08178100018995, + "p95": 127.91637400005129, + "max": 281.6627239990339 + } + }, + { + "start": 29856.063789742, + "first": 29856.398485387, + "last": 29868.553098368, + "end": 29868.553440712, + "elapsed_s": 12.489650970001094, + "chunk_offsets_s": [ + 0.33469564500046545, + 0.44915714600210777, + 0.5779859419999411, + 0.7060523170002853, + 0.8190706050008885, + 0.9196176559999003, + 1.0191161970033136, + 1.128447049002716, + 1.2476885750002111, + 1.3513661960023455, + 1.4505541250000533, + 1.5467242570011877, + 1.6483412180023151, + 1.7529999440012034, + 1.8614424680017692, + 1.988580182001897, + 2.100017689001106, + 2.215456804002315, + 2.3165391640031885, + 2.41593410600035, + 2.5271902439999394, + 2.6309338490027585, + 2.734926326000277, + 2.8469021190030617, + 2.950797549001436, + 3.0639367180010595, + 3.175935750001372, + 3.27632730500045, + 3.395360640002764, + 3.5165993770024215, + 3.6291179489999195, + 3.729249009000341, + 3.8277057140003308, + 3.935572480000701, + 4.055732630000421, + 4.183169322001049, + 4.295846945002268, + 4.408596332999878, + 4.524521456001821, + 4.6239394870026445, + 4.730582927000796, + 4.8415393690011115, + 4.958823373002815, + 5.077402982002241, + 5.192626389001816, + 5.297407493002538, + 5.402735200001189, + 5.517321512001217, + 5.635367610000685, + 5.745911499001522, + 5.857325643002696, + 5.975195617000281, + 6.100043200000073, + 6.22591210900282, + 6.366278117002366, + 6.493310140001995, + 6.603723508000257, + 6.716017284001282, + 6.835290312003053, + 6.943074400001933, + 7.062466462000884, + 7.182987722000689, + 7.293371446001402, + 7.411339532001875, + 7.528746772000886, + 7.638135517001501, + 7.7385732440016, + 7.839921183000115, + 7.956464534003317, + 8.080046803002915, + 8.183057269001438, + 8.288581453001825, + 8.396379296002124, + 8.507996587002708, + 8.605208677003247, + 8.7173816909999, + 8.836777073000121, + 8.975874593001208, + 9.110237191001943, + 9.214101234003465, + 9.30866305300151, + 9.401864552000916, + 9.507327276001888, + 9.608830677003425, + 9.702849033001257, + 9.81656995600133, + 9.933446260001801, + 10.030288091002149, + 10.129373165000288, + 10.227010950002295, + 10.328458777003107, + 10.423387160000857, + 10.51599675700345, + 10.616896209001425, + 10.714203605002695, + 10.821254567003052, + 10.941932514000655, + 11.058764478002558, + 11.166254376003053, + 11.27290053100296, + 11.391466460001539, + 11.487173313002131, + 11.581605715000478, + 11.672631244000513, + 11.776177601001109, + 11.883424461000686, + 11.995809081003245, + 12.071548045001691, + 12.152608151001914, + 12.240520454000944, + 12.314915060000203, + 12.386509572002979, + 12.438407577999897, + 12.48930862600173 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 108.44252400056575, + "p95": 127.43669200062868, + "max": 140.36600799954613 + } + }, + { + "start": 29856.063182283, + "first": 29856.150063438, + "last": 29867.455062832, + "end": 29867.455292165, + "elapsed_s": 11.392109881999204, + "chunk_offsets_s": [ + 0.0868811549989914, + 0.45007848200111766, + 0.5780578849989979, + 0.706420259000879, + 0.8194602980001946, + 0.92041732099824, + 1.0196163569999044, + 1.1295147349992476, + 1.2480302529984328, + 1.3520413410005858, + 1.4513200350011175, + 1.5474545549986942, + 1.6484609690014622, + 1.7537095320003573, + 1.8621182040005806, + 1.988947884998197, + 2.101420921000681, + 2.2164586289982253, + 2.3168680380003934, + 2.4159926710017317, + 2.527258267000434, + 2.6316455009982747, + 2.735372325998469, + 2.846954841999832, + 2.951642363001156, + 3.0652680050006893, + 3.1764180489990395, + 3.2761692739986756, + 3.3952974870007893, + 3.5164955549989827, + 3.6292606530005287, + 3.729554741999891, + 3.8279870379992644, + 3.935961502000282, + 4.055775607001124, + 4.183227648998582, + 4.29608846499832, + 4.4086858369992115, + 4.524877955998818, + 4.6242230360003305, + 4.730926704000012, + 4.841589182000462, + 4.95919375000085, + 5.07782723100172, + 5.192661028999282, + 5.297391173000506, + 5.40284853599951, + 5.517281552998611, + 5.636258629998338, + 5.746207437998237, + 5.857575678001012, + 5.975969959999929, + 6.100759861001279, + 6.225799823998386, + 6.367198489999282, + 6.493410416998813, + 6.604253102999792, + 6.716029870000057, + 6.835569957998814, + 6.943425156001467, + 7.063503157998639, + 7.183496893998381, + 7.293758133000665, + 7.412328916998376, + 7.529002713999944, + 7.638826441001584, + 7.738678357000026, + 7.840776367000217, + 7.956725709998864, + 8.080454120001377, + 8.18421049600147, + 8.289524935000372, + 8.397051879001083, + 8.508268679001048, + 8.605571692998637, + 8.717813639999804, + 8.837132726999698, + 8.976394713001355, + 9.110695179999311, + 9.214629804999277, + 9.308923974000209, + 9.402030738998292, + 9.50773643499997, + 9.609313026001473, + 9.70326051200027, + 9.816733968000335, + 9.933812685998419, + 10.031060191999131, + 10.129759324998304, + 10.227805186001206, + 10.329215576999559, + 10.424275489000138, + 10.516827531999297, + 10.617722629998752, + 10.715054930999031, + 10.82109315300113, + 10.942131961000996, + 11.05949242099814, + 11.166605709000578, + 11.27340402199843, + 11.391880548999325 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 110.75258200071403, + "p95": 127.97940299788024, + "max": 363.19732700212626 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 87.0, + "per_stream_tps": 18.1, + "ttft_s_max": 0.42, + "request": { + "ttft_s": 0.249, + "decode_tps": 18.126, + "prefill_tps": null, + "itl_mean_ms": 55.169, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.307, + "round_gap_p50_ms": 126.23, + "round_gap_p95_ms": 146.113, + "tokens_per_chunk": 2.286 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 54.339, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.178, + "round_gap_p50_ms": 126.342, + "round_gap_p95_ms": 145.975, + "tokens_per_chunk": 2.315 + }, + "timing_valid": true, + "end_to_end_tps": 86.81105898202487, + "makespan_s": 14.744665195998095, + "steady_tps": 90.8, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 86.98047589227765, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 14.744195473998843, + "end_to_end_tps": 86.81382461710169, + "timing_valid": true, + "steady_tps": 91.73228161512587, + "per_stream_tps": [ + 18.1, + 18.9, + 17.7, + 20.1, + 17.5 + ], + "ttft_s": [ + 0.25, + 0.33, + 0.09, + 0.42, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.24860577300205478, + "decode_tps": 18.12625405955774, + "prefill_tps": null, + "itl_mean_ms": 55.168596705876624, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.30708599966601, + "round_gap_p50_ms": 126.22994399862364, + "round_gap_p95_ms": 145.50176700140582, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.32800730300004943, + "decode_tps": 18.877946829573435, + "prefill_tps": null, + "itl_mean_ms": 52.97186230196601, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.32406200203695, + "round_gap_p50_ms": 126.48794000051566, + "round_gap_p95_ms": 146.1134819983272, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08530810899901553, + "decode_tps": 17.731447459658668, + "prefill_tps": null, + "itl_mean_ms": 56.39697505097252, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.67347399759456, + "round_gap_p50_ms": 126.29075000222656, + "round_gap_p95_ms": 145.2551959991979, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.41724371099917335, + "decode_tps": 20.050966523311683, + "prefill_tps": null, + "itl_mean_ms": 49.87290756469912, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.11036100055207, + "round_gap_p50_ms": 126.72621599995182, + "round_gap_p95_ms": 144.7201410010166, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.169809669998358, + "decode_tps": 17.496804347799973, + "prefill_tps": null, + "itl_mean_ms": 57.1532938313812, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.0466189988947, + "round_gap_p50_ms": 126.15769900003215, + "round_gap_p95_ms": 144.89096399847767, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 54.312727090979095, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.17755299937562, + "round_gap_p50_ms": 126.342024999758, + "round_gap_p95_ms": 145.2551959991979, + "round_gap_mean_ms": 126.3662902207999, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 29868.555784523, + "first": 29868.804390296, + "last": 29882.872382456, + "end": 29882.872498793, + "elapsed_s": 14.316714270000375, + "chunk_offsets_s": [ + 0.24860577300205478, + 0.5480866569996579, + 0.6922442460017919, + 0.8353066410018073, + 0.9617691019993799, + 1.0734030419989722, + 1.182025293001061, + 1.3008219200019084, + 1.4336657599997125, + 1.5570127889986907, + 1.6723777490005887, + 1.7807042989988986, + 1.896214903001237, + 2.0155937999988964, + 2.1461891910003033, + 2.286683490001451, + 2.4255745020018367, + 2.5712763590017857, + 2.6863182460001553, + 2.801449776001391, + 2.928463758998987, + 3.044331731998682, + 3.1630616469992674, + 3.2952760449989, + 3.418687861001672, + 3.5496638900003745, + 3.6765728880018287, + 3.791303711001092, + 3.9240696470005787, + 4.053944469000271, + 4.180735639998602, + 4.3008416869997745, + 4.415740424999967, + 4.537650722999388, + 4.663080639002146, + 4.805370594000124, + 4.931600537998747, + 5.0526851680006075, + 5.179443508000986, + 5.292824447002204, + 5.41496137699869, + 5.543344386998797, + 5.669293271999777, + 5.797855967000942, + 5.9340099210021435, + 6.055165490000945, + 6.166787083999225, + 6.298848941001779, + 6.4385051319986815, + 6.5737643700012995, + 6.695954077000351, + 6.819966110000678, + 6.959581409999373, + 7.099888495999039, + 7.254530995000096, + 7.394716391001566, + 7.534490691999963, + 7.659894839998742, + 7.792436946001544, + 7.907683100002032, + 8.04121515999941, + 8.186716927000816, + 8.314266031000443, + 8.447719386000244, + 8.588276889000554, + 8.714686224000616, + 8.833485277998989, + 8.960129303999565, + 9.096278788001655, + 9.236106042000756, + 9.358949751000182, + 9.499912799001322, + 9.630067221998615, + 9.74902160900092, + 9.865308709999226, + 9.999148393999349, + 10.13511877700148, + 10.292303673999413, + 10.451759343999584, + 10.578559885001596, + 10.684426002000691, + 10.791247019002185, + 10.922728674999234, + 11.057865468999807, + 11.169683899999654, + 11.29620069700104, + 11.435843191000458, + 11.548014193998824, + 11.656185324001854, + 11.768790228001308, + 11.898310970998864, + 12.020335952001915, + 12.13113956899906, + 12.25706382700082, + 12.377292035998835, + 12.507864594001148, + 12.642526375999296, + 12.768097353000485, + 12.88400492800065, + 13.003282304001914, + 13.135018764998676, + 13.246412004002195, + 13.359535783001775, + 13.464182076000725, + 13.590154960998916, + 13.707181617999595, + 13.836234633999993, + 13.93758793500092, + 14.040999491000548, + 14.141795321000245, + 14.229521229000966, + 14.316597933000594 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 126.22994399862364, + "p95": 145.50176700140582, + "max": 299.4808839976031 + } + }, + { + "start": 29868.556303896, + "first": 29868.884311199, + "last": 29882.392136086, + "end": 29882.39228813, + "elapsed_s": 13.835984233999625, + "chunk_offsets_s": [ + 0.32800730300004943, + 0.5474222330012708, + 0.6918814640011988, + 0.8354086550025386, + 0.9609273919995758, + 1.0734196539997356, + 1.1815820709998661, + 1.3000443420023657, + 1.433375127002364, + 1.556204754000646, + 1.6719365760000073, + 1.7803498550019867, + 1.8954665010023746, + 2.0152118440018967, + 2.145920325001498, + 2.286548888001562, + 2.4245869250007672, + 2.5710235770020518, + 2.6858377160024247, + 2.8016934459992626, + 2.9284879560000263, + 3.0438947200018447, + 3.1624199649995717, + 3.295006442000158, + 3.4182250810008554, + 3.549214345999644, + 3.67564918700009, + 3.790954931999295, + 3.9238909670020803, + 4.053650027002732, + 4.180061917002604, + 4.300687798000581, + 4.415287055002409, + 4.537336003002565, + 4.6631227120014955, + 4.804395439001382, + 4.930883379001898, + 5.051913529001467, + 5.179088056000182, + 5.291992966001999, + 5.414278980002564, + 5.542925492001814, + 5.669287698001426, + 5.797207804000209, + 5.933894107001834, + 6.055044443000952, + 6.165970407000714, + 6.298275135999575, + 6.438284839001426, + 6.573419112002739, + 6.6954920000025595, + 6.8191103499993915, + 6.958149353002227, + 7.099454908002372, + 7.2542580619992805, + 7.394263726000645, + 7.533332139999402, + 7.6596509869996225, + 7.792284608000045, + 7.907199577999563, + 8.040809244001139, + 8.186922725999466, + 8.314045721999719, + 8.447280628002773, + 8.587818846001028, + 8.714469835002092, + 8.832693889002257, + 8.960002862000692, + 9.096418663000804, + 9.235908091002784, + 9.35896789400067, + 9.499291956002708, + 9.62972436400014, + 9.748749747999682, + 9.864301947000058, + 9.998564618999808, + 10.134728162000101, + 10.291620301999501, + 10.45116851000057, + 10.577821793001931, + 10.684074550001242, + 10.790538744000514, + 10.921915360999265, + 11.057236560001911, + 11.168739603999711, + 11.296225737001805, + 11.434681152000849, + 11.547321203001047, + 11.655528055001923, + 11.768482340001356, + 11.898367460002191, + 12.019380837002245, + 12.130516214001545, + 12.257054215999233, + 12.376080144000298, + 12.507157631000155, + 12.642427156002668, + 12.768169991002651, + 12.883129821002512, + 13.002505353000743, + 13.13433416100088, + 13.245621049001784, + 13.359055012999306, + 13.463735060002364, + 13.590078179000557, + 13.707141470000352, + 13.835832190001383 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 126.51297099910153, + "p95": 146.1134819983272, + "max": 219.41493000122136 + } + }, + { + "start": 29868.555443476, + "first": 29868.640751585, + "last": 29883.021980223, + "end": 29883.02214435, + "elapsed_s": 14.466700873999798, + "chunk_offsets_s": [ + 0.08530810899901553, + 0.5481978269999672, + 0.6933155599981546, + 0.8353914349972911, + 0.9615265020001971, + 1.0736621289979666, + 1.1819167809990176, + 1.3009855860000243, + 1.4336354959996243, + 1.5572198279987788, + 1.6719437319989083, + 1.7808333299981314, + 1.8956082099975902, + 2.0156091499993636, + 2.1462816189996374, + 2.2870924459966773, + 2.42554310199921, + 2.570798297998408, + 2.686462865996873, + 2.801533865997044, + 2.9290247760000057, + 3.044548998997925, + 3.163037752998207, + 3.2952609859967197, + 3.418965192999167, + 3.549896552998689, + 3.676204843999585, + 3.7911764589989616, + 3.9251437279999664, + 4.054130221997184, + 4.1804209719994105, + 4.301106823997543, + 4.415995846997248, + 4.537729404997663, + 4.664028392999171, + 4.805122154997662, + 4.931415636998281, + 5.052471010996669, + 5.180006716997013, + 5.292752011999255, + 5.415003999998589, + 5.543241510997177, + 5.669236057998205, + 5.798113324999576, + 5.933941678998963, + 6.055085148997023, + 6.1669083059969125, + 6.29887040599715, + 6.438895713999955, + 6.574007788000017, + 6.695861716998479, + 6.8196378409993486, + 6.958766949999699, + 7.100174806997529, + 7.254791985997144, + 7.394818471999315, + 7.5345900199972675, + 7.660436664998997, + 7.793325896996976, + 7.907958992000204, + 8.041319402000227, + 8.186970971997653, + 8.314399721999507, + 8.448251128000265, + 8.58892460199786, + 8.715285585996753, + 8.83314740700007, + 8.960281271996791, + 9.096856655996817, + 9.236897396000131, + 9.359027008998964, + 9.500080451998656, + 9.630799965998449, + 9.74901897199743, + 9.86495944499984, + 9.999367997999798, + 10.134932902998116, + 10.292357981998066, + 10.45188019899797, + 10.578349380997679, + 10.684678079996957, + 10.79113177399995, + 10.922479503999057, + 11.057789659000264, + 11.169466528997873, + 11.296408233996772, + 11.435916448997887, + 11.547997761997976, + 11.656078961997991, + 11.768964218998008, + 11.898365614997601, + 12.019980989996839, + 12.131821145998401, + 12.257306332998269, + 12.37621102599951, + 12.50825909699779, + 12.64229871599673, + 12.768353799998295, + 12.883712336999452, + 13.003048241997021, + 13.134532661999401, + 13.246101487999113, + 13.359325359997456, + 13.464217242999439, + 13.590587073998904, + 13.707643531997746, + 13.836062221998873, + 13.938086194997595, + 14.041456058999756, + 14.141342619997886, + 14.229337564996968, + 14.316820577998442, + 14.393617239999003, + 14.466536746997008 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 126.29075000222656, + "p95": 145.2551959991979, + "max": 462.8897180009517 + } + }, + { + "start": 29868.556055149, + "first": 29868.97329886, + "last": 29881.690890289, + "end": 29881.691060226, + "elapsed_s": 13.135005076997913, + "chunk_offsets_s": [ + 0.41724371099917335, + 0.5478776499985543, + 0.6925977909995709, + 0.8353114579986141, + 0.9613343149976572, + 1.0733866049995413, + 1.1815182119971723, + 1.3009787819974008, + 1.433175307996862, + 1.5569540500000585, + 1.6718181889991683, + 1.7804927949982812, + 1.8951332179967721, + 2.0155078209973, + 2.14641552899775, + 2.286525889998302, + 2.4250751829968067, + 2.570651354999427, + 2.686155395997048, + 2.801447277997795, + 2.9284548910000012, + 3.044214319997991, + 3.162515818999964, + 3.29533329199694, + 3.4182287309995445, + 3.5495343939983286, + 3.676222381996922, + 3.7913595659993007, + 3.923992261999956, + 4.053723633998743, + 4.180537114996696, + 4.301048273999186, + 4.415665536998858, + 4.53717670900005, + 4.663000212996849, + 4.805247275999136, + 4.931589300998894, + 5.052355644998897, + 5.17911830599769, + 5.2924250309988565, + 5.4147504169995955, + 5.543296546999045, + 5.668897488998482, + 5.797647310999309, + 5.933900205996906, + 6.055053838997992, + 6.166876307997882, + 6.298177973996644, + 6.438178857999446, + 6.573723698998947, + 6.695960443998047, + 6.819151035997493, + 6.959046625997871, + 7.099831356998038, + 7.254805247997865, + 7.394338869999046, + 7.53361760199914, + 7.659531353998318, + 7.792256185999577, + 7.907508278000023, + 8.041253503997723, + 8.187005855997995, + 8.313942221997422, + 8.447935525997309, + 8.588113078996685, + 8.714557334998972, + 8.832997339999565, + 8.960118033999606, + 9.09579608399872, + 9.236532707000151, + 9.35893114999999, + 9.499359530997026, + 9.629528085999482, + 9.748686979997728, + 9.865165464998427, + 9.99870490199828, + 10.134543262996885, + 10.29199213999891, + 10.451194296998437, + 10.57792051299839, + 10.68389688499883, + 10.790709580000112, + 10.921960160998424, + 11.057304726997245, + 11.16921156799799, + 11.296885242998542, + 11.43562198699874, + 11.54792493199784, + 11.655547727998055, + 11.769402512996749, + 11.898505230998126, + 12.019873766999808, + 12.13100240499989, + 12.257184368998423, + 12.375708231997123, + 12.507521531999373, + 12.641989065999951, + 12.767985460999626, + 12.883611754998128, + 13.002565085997048, + 13.134835139997449 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 126.74443849937234, + "p95": 144.7201410010166, + "max": 159.20215699952678 + } + }, + { + "start": 29868.555313981, + "first": 29868.725123651, + "last": 29883.299213578, + "end": 29883.299509455, + "elapsed_s": 14.744195473998843, + "chunk_offsets_s": [ + 0.169809669998358, + 0.5484818170007202, + 0.69479967100051, + 0.8354488279983343, + 0.9616980739992869, + 1.0737251079990529, + 1.182100548001472, + 1.301657083000464, + 1.4338191189999634, + 1.5572646250002435, + 1.6721062290016562, + 1.78091983700142, + 1.8954058300005272, + 2.015636325999367, + 2.1473003719984263, + 2.287092686001415, + 2.4258573560000514, + 2.5707402970001567, + 2.6866840690017852, + 2.801530456999899, + 2.928612332998455, + 3.0446000379997713, + 3.163749325001845, + 3.295548975998827, + 3.4194449999995413, + 3.549782966998464, + 3.6764702030013723, + 3.7915461849988787, + 3.9251795310010493, + 4.0543479830012075, + 4.180796667998948, + 4.301847924001777, + 4.416356305999216, + 4.537735423000413, + 4.663893122000445, + 4.805304673998762, + 4.931633814998349, + 5.05256627199924, + 5.179974964001303, + 5.293059896001068, + 5.4152023479982745, + 5.5435913750006875, + 5.669471247998445, + 5.798128866001207, + 5.934205149998888, + 6.055437637998693, + 6.1673431230010465, + 6.29949423799917, + 6.438794261001021, + 6.573934775999078, + 6.6968425330014725, + 6.819682063000073, + 6.958735676998913, + 7.100212179000664, + 7.255155037000804, + 7.3953328159986995, + 7.53421556000103, + 7.660420214000624, + 7.7927766339998925, + 7.907995387999108, + 8.042403272000229, + 8.187294235998706, + 8.31459144299879, + 8.44851022900184, + 8.588556848000735, + 8.715060578000703, + 8.833524270001362, + 8.960941609999281, + 9.097240464998322, + 9.236417354000878, + 9.359338333000778, + 9.500543515001482, + 9.630343083001208, + 9.749060344998725, + 9.86497671500183, + 9.999288055998477, + 10.135113612999703, + 10.292668106001656, + 10.452054986999428, + 10.578737863001152, + 10.685012898000423, + 10.791324887999508, + 10.922542226999212, + 11.057853839000018, + 11.169649848001427, + 11.297044627001014, + 11.43577020000157, + 11.548246362999635, + 11.656349217999377, + 11.770367714001623, + 11.899032224999246, + 12.020880803000182, + 12.131401058999472, + 12.257595538001624, + 12.37823639400085, + 12.5081944880003, + 12.643222759001219, + 12.768937730001198, + 12.884041410001373, + 13.003420959001232, + 13.135129684000276, + 13.246663860001718, + 13.359723156001564, + 13.464292739001394, + 13.591305490001105, + 13.707894758001203, + 13.836301171999366, + 13.938473556001554, + 14.041996568001196, + 14.141819078999106, + 14.229813384001318, + 14.316892185001052, + 14.394003880999662, + 14.466806426000403, + 14.520625163000659, + 14.576916878999327, + 14.640530830998614, + 14.687695747001271, + 14.743899597000564 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 126.1760890010919, + "p95": 144.89096399847767, + "max": 378.6721470023622 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 86.77555790451888, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 14.779704777000006, + "end_to_end_tps": 86.6052481638145, + "timing_valid": true, + "steady_tps": 90.68680125190092, + "per_stream_tps": [ + 18.0, + 18.7, + 18.0, + 20.0, + 17.4 + ], + "ttft_s": [ + 0.17, + 0.25, + 0.33, + 0.41, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16643212900089566, + "decode_tps": 17.975939457544822, + "prefill_tps": null, + "itl_mean_ms": 55.62991588627554, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.6691030022339, + "round_gap_p50_ms": 125.77784799941583, + "round_gap_p95_ms": 152.30388499912806, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2462028250010917, + "decode_tps": 18.71287355841507, + "prefill_tps": null, + "itl_mean_ms": 53.439146952943844, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.83906999783358, + "round_gap_p50_ms": 126.2220390017319, + "round_gap_p95_ms": 151.83728900228743, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33069611900282325, + "decode_tps": 18.006111159736346, + "prefill_tps": null, + "itl_mean_ms": 55.53670035293965, + "itl_p50_ms": 0.0, + "itl_p95_ms": 140.08400400052778, + "round_gap_p50_ms": 125.58776999867405, + "round_gap_p95_ms": 151.85439799824962, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.413455909001641, + "decode_tps": 19.96795056388722, + "prefill_tps": null, + "itl_mean_ms": 50.08025219215722, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.95046499965247, + "round_gap_p50_ms": 126.71740299992962, + "round_gap_p95_ms": 146.29550400059088, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.0865523940010462, + "decode_tps": 17.355111580903774, + "prefill_tps": null, + "itl_mean_ms": 57.61991188234841, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.5364720010548, + "round_gap_p50_ms": 125.32238999847323, + "round_gap_p95_ms": 152.2298009986116, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 54.46118545333293, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.58069399814121, + "round_gap_p50_ms": 126.01195699971868, + "round_gap_p95_ms": 151.85439799824962, + "round_gap_mean_ms": 126.7116997317509, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 29883.301449081, + "first": 29883.46788121, + "last": 29897.653509761, + "end": 29897.653714265, + "elapsed_s": 14.352265184003045, + "chunk_offsets_s": [ + 0.16643212900089566, + 0.535759163001785, + 0.6817958880019432, + 0.8340997730010713, + 0.9592919030001212, + 1.0722404010011815, + 1.1819490640009462, + 1.302306313002191, + 1.4286358550016303, + 1.5454708040015248, + 1.6610196770016046, + 1.778627696003241, + 1.8960184210009174, + 2.01580024100258, + 2.147139873002743, + 2.286347036002553, + 2.4217529260022275, + 2.56063390500276, + 2.677626730001066, + 2.799401950000174, + 2.9296477130010317, + 3.0443925890031096, + 3.1643894650005677, + 3.2957224320016394, + 3.41630164900198, + 3.537982658002875, + 3.6661032420015545, + 3.7882370950028417, + 3.921650808002596, + 4.052414091001992, + 4.180890824001835, + 4.302022090003447, + 4.414408853001078, + 4.528499298001407, + 4.653374186000292, + 4.806569664000563, + 4.933311718003097, + 5.054417993000243, + 5.183168421001028, + 5.297182536000037, + 5.4181878060007875, + 5.538910462000786, + 5.6664078870016965, + 5.804524169001525, + 5.940636111001368, + 6.063307060001534, + 6.177903194002283, + 6.30837607900321, + 6.446333837000566, + 6.577307519000897, + 6.7030853670003125, + 6.8334820449999825, + 6.974758687003487, + 7.1193206180032576, + 7.27727150300052, + 7.416940606002754, + 7.546329105000041, + 7.672767438001756, + 7.813625228001911, + 7.926302895000845, + 8.061983973002498, + 8.20954080500087, + 8.334797479001281, + 8.464487714001734, + 8.599914423000882, + 8.733987061001244, + 8.855622163002408, + 8.981845362002787, + 9.119697887002985, + 9.261835304001579, + 9.38469225100198, + 9.518495609001548, + 9.648184046000097, + 9.773762523000187, + 9.89208869200229, + 10.028138580000814, + 10.16461380400142, + 10.323645533000672, + 10.478886654000235, + 10.602774973001942, + 10.715768691999983, + 10.828477533003024, + 10.961080638000567, + 11.096501549000095, + 11.21028280000246, + 11.337921757000004, + 11.473899432003236, + 11.578450527002133, + 11.688596607000363, + 11.808551006000926, + 11.93860659100028, + 12.060700000001816, + 12.172348704003525, + 12.298587182001938, + 12.418894736001675, + 12.544889865002915, + 12.681452120003087, + 12.816144562002592, + 12.932951335002144, + 13.052845842001261, + 13.18501560200093, + 13.29622530600318, + 13.407980952000798, + 13.508177499003068, + 13.632172337001975, + 13.75538515000153, + 13.87362812700303, + 13.974351818000287, + 14.077123242001107, + 14.176659724002093, + 14.26448246000291, + 14.35206068000116 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 125.77784799941583, + "p95": 152.30388499912806, + "max": 369.3270340008894 + } + }, + { + "start": 29883.301792059, + "first": 29883.547994884, + "last": 29897.174977357, + "end": 29897.175573938, + "elapsed_s": 13.87378187899958, + "chunk_offsets_s": [ + 0.2462028250010917, + 0.5358613680000417, + 0.6816299519996392, + 0.8334672410019266, + 0.9594966969998495, + 1.071609629001614, + 1.181723110999883, + 1.3014728750022186, + 1.42808260899983, + 1.5452980820009543, + 1.6603078590014775, + 1.7775888559990562, + 1.8957300350011792, + 2.0150778600000194, + 2.146493193002243, + 2.2863322630000766, + 2.421598434000771, + 2.5606529819997377, + 2.6770997480016376, + 2.799415025001508, + 2.928404432001116, + 3.0435009120010363, + 3.1639823890000116, + 3.2961115180005436, + 3.4146890480005823, + 3.5375369630019122, + 3.666016753002623, + 3.788063477000833, + 3.921231103999162, + 4.052251036002417, + 4.179754838001827, + 4.301186605000112, + 4.414630547002162, + 4.528241031999642, + 4.653550347000419, + 4.80638954800088, + 4.93289188700146, + 5.054394112001319, + 5.183025738999277, + 5.297094574001676, + 5.418498467999598, + 5.538951143000304, + 5.666279449000285, + 5.804424523001217, + 5.940584639000008, + 6.063326023002446, + 6.177831971999694, + 6.307942628001911, + 6.445782800001325, + 6.576507933001267, + 6.7028353749992675, + 6.833188921002147, + 6.974548418002087, + 7.118185225001071, + 7.276861210000789, + 7.416523479001626, + 7.5463084090006305, + 7.672475338000368, + 7.8131558150016645, + 7.92601111499971, + 8.061971952000022, + 8.208791506000125, + 8.334229126001446, + 8.46401132200117, + 8.599652140001126, + 8.733160523999686, + 8.855113555000571, + 8.981735137000214, + 9.119278792000841, + 9.261594065999816, + 9.384270595000999, + 9.518293741999514, + 9.647738159001165, + 9.772908073999133, + 9.891569392002566, + 10.027726991000236, + 10.164583681002114, + 10.323542318001273, + 10.47848354800226, + 10.602348516000347, + 10.715507192999212, + 10.828056484999252, + 10.961073833001137, + 11.096280173002015, + 11.210238032999769, + 11.33791168800235, + 11.473143074999825, + 11.577752634002536, + 11.688331442001072, + 11.808612444001483, + 11.938547836001817, + 12.060063244000048, + 12.172264790002373, + 12.298444324002048, + 12.418927078000706, + 12.545149117002438, + 12.680697748000966, + 12.815765638002631, + 12.93505437800195, + 13.0528778959997, + 13.185795575001976, + 13.295939207000629, + 13.40757350900094, + 13.507945720000862, + 13.631911125001352, + 13.754897459999484, + 13.873185298001772 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 126.27474049986631, + "p95": 151.83728900228743, + "max": 289.65854299895 + } + }, + { + "start": 29883.301533649, + "first": 29883.632229768, + "last": 29897.794088358, + "end": 29897.794179568, + "elapsed_s": 14.492645919002825, + "chunk_offsets_s": [ + 0.33069611900282325, + 0.5364286820004054, + 0.6817878210022172, + 0.8336422190004669, + 0.9590955919993576, + 1.072278556002857, + 1.18129915100144, + 1.3015020359998744, + 1.4282092050016217, + 1.5456740129993705, + 1.6611380630019994, + 1.7782742519993917, + 1.8956523170018045, + 2.0155429360020207, + 2.1466259190019628, + 2.2867099230024905, + 2.422330803001387, + 2.5605984550020366, + 2.6771399040007964, + 2.7995567140023923, + 2.9288491510014865, + 3.04402779800148, + 3.1643801980026183, + 3.2958501650027756, + 3.4151236190009513, + 3.5379372050010716, + 3.6657909140012634, + 3.788623140000709, + 3.921894107999833, + 4.052844019999611, + 4.181096706000972, + 4.301889893002226, + 4.414030746000208, + 4.52806159100146, + 4.653721916001814, + 4.806936263001262, + 4.933767927002918, + 5.054391681002016, + 5.182602597000368, + 5.297903940001561, + 5.418157620999409, + 5.538920975999645, + 5.666282026999397, + 5.80473185899973, + 5.940501160999702, + 6.063150663001579, + 6.177736907000508, + 6.308243498002412, + 6.446297875001619, + 6.577141153000412, + 6.702754186000675, + 6.833519159001298, + 6.975046335002844, + 7.118782354002178, + 7.277244247001363, + 7.416731379002158, + 7.546651882999868, + 7.672619044002204, + 7.81368842899974, + 7.926529134001612, + 8.062143169001502, + 8.209157178000169, + 8.334425615001237, + 8.46406385100272, + 8.600136100001691, + 8.73367286100256, + 8.855438111000694, + 8.98217587000181, + 9.119678985000064, + 9.261812561002444, + 9.384869084002275, + 9.518762339001114, + 9.648170986001787, + 9.77336822000143, + 9.891763593001087, + 10.027865112002473, + 10.165079325000988, + 10.324165501002426, + 10.47936901200228, + 10.603019231999497, + 10.716355255000963, + 10.82903642400197, + 10.961082846002682, + 11.096491018000961, + 11.210380443000759, + 11.33830664800189, + 11.472945518002234, + 11.578292162001162, + 11.688906580002367, + 11.808523654002784, + 11.938443585000641, + 12.061062356002367, + 12.172673994002253, + 12.298261764000927, + 12.419311091001873, + 12.544939974002773, + 12.681068781999784, + 12.81610336599988, + 12.932536388001608, + 13.053179294001893, + 13.18507286500244, + 13.296479599001032, + 13.408126160000393, + 13.507885914001236, + 13.631921035001142, + 13.75534104200051, + 13.873617740002373, + 13.974556193999888, + 14.077089247999538, + 14.176703384000575, + 14.264348614000482, + 14.352032671002235, + 14.428231540001434, + 14.492554709002434 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 125.58776999867405, + "p95": 151.85439799824962, + "max": 205.73256299758214 + } + }, + { + "start": 29883.302444421, + "first": 29883.71590033, + "last": 29896.486364639, + "end": 29896.487044347, + "elapsed_s": 13.184599926000374, + "chunk_offsets_s": [ + 0.413455909001641, + 0.5351156520009681, + 0.680629955000768, + 0.8326017620020139, + 0.9586189590008871, + 1.0708557819998532, + 1.1802965610004321, + 1.3009767340008693, + 1.4277325090006343, + 1.544230930001504, + 1.6601008550023835, + 1.7778025640000124, + 1.8945955199997115, + 2.0139578289999918, + 2.1462984650024737, + 2.285248930002126, + 2.4214708450017497, + 2.560206805999769, + 2.676085620001686, + 2.7984553759997652, + 2.9284223080030642, + 3.0425728400005028, + 3.1631085720000556, + 3.295063444002153, + 3.4155043810023926, + 3.5367832760020974, + 3.6643831380024494, + 3.786964941002225, + 3.9211501030004, + 4.051802123001835, + 4.179323675001797, + 4.301624583000375, + 4.413323680000758, + 4.52785636200133, + 4.652474713002448, + 4.805411686000298, + 4.932773739001277, + 5.05427790199974, + 5.183431884001038, + 5.296936128001107, + 5.4174379910000425, + 5.538624745000561, + 5.665192133001256, + 5.803679911001382, + 5.939402367999719, + 6.0630748300027335, + 6.177082591002545, + 6.307212004001485, + 6.44559652099997, + 6.576111243000923, + 6.702398977002304, + 6.833167408000008, + 6.973520302999532, + 7.118196858002193, + 7.276020027002232, + 7.415600721000374, + 7.54547262000051, + 7.672190023000439, + 7.812880822002626, + 7.925683116001892, + 8.061648241000512, + 8.207943745001103, + 8.333177197000623, + 8.463238623000507, + 8.599171153000498, + 8.732426491002116, + 8.85431337600312, + 8.980783700000757, + 9.119127786001627, + 9.261023311002646, + 9.38387846600017, + 9.518018093000137, + 9.64692248800202, + 9.772934445001738, + 9.890759050002089, + 10.027325275001203, + 10.164098501001718, + 10.32307168600164, + 10.47838809300083, + 10.601609168999858, + 10.7153149270016, + 10.828056929000013, + 10.959921796002163, + 11.095290629000374, + 11.20995190100075, + 11.338273399000173, + 11.47295404200122, + 11.577977283999644, + 11.687423008999758, + 11.807404424002016, + 11.938318599000922, + 12.0599805439997, + 12.172002344002976, + 12.298310458001652, + 12.417694571002357, + 12.544742007001332, + 12.680605285000638, + 12.815372484001273, + 12.932921782001358, + 13.052037770001334, + 13.183920218001731 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 126.7365889998473, + "p95": 146.29550400059088, + "max": 158.97318499992252 + } + }, + { + "start": 29883.301357679, + "first": 29883.387910073, + "last": 29898.080987603, + "end": 29898.081062456, + "elapsed_s": 14.779704777000006, + "chunk_offsets_s": [ + 0.0865523940010462, + 0.5361168009985704, + 0.6817756900018139, + 0.8340054910004255, + 0.9600376839989622, + 1.0725385660007305, + 1.1815781900004367, + 1.3014968339994084, + 1.4280574980002712, + 1.5452323040008196, + 1.6606310710012622, + 1.778333324000414, + 1.8959950380012742, + 2.0146570049982984, + 2.1465593239990994, + 2.2860957960001542, + 2.422436224998819, + 2.560906831000466, + 2.6774360359995626, + 2.799665031998302, + 2.929220472000452, + 3.044036983999831, + 3.1640382779987704, + 3.2958971059997566, + 3.4152239200011536, + 3.537720439999248, + 3.6653634979993512, + 3.7884080879994144, + 3.9223953379987506, + 4.052823166999588, + 4.180765348999557, + 4.3014626360018156, + 4.414836222000304, + 4.528462104000937, + 4.65378449399941, + 4.806295955000678, + 4.934142336998775, + 5.055080307000026, + 5.183697593998659, + 5.297828334998485, + 5.418701550999685, + 5.538885846999619, + 5.666958458001318, + 5.8044881589994475, + 5.940597988999798, + 6.06410869300089, + 6.178115589998924, + 6.3085481119996984, + 6.446728012000676, + 6.577249424000911, + 6.703027398998529, + 6.835152290001133, + 6.974505687998317, + 7.119094795001729, + 7.276891476998571, + 7.417262921000656, + 7.546231958000135, + 7.67312569799833, + 7.813465504001215, + 7.926660354998603, + 8.061767464998411, + 8.208628167001734, + 8.334213268000894, + 8.46417517900045, + 8.599806920999981, + 8.734134767000796, + 8.855204334999144, + 8.981790243000432, + 9.121082210000168, + 9.261796814000263, + 9.384616791001463, + 9.519352663999598, + 9.64786680600082, + 9.773773133998475, + 9.891744035998272, + 10.027748024000175, + 10.164567215000716, + 10.324535173000186, + 10.479188099001476, + 10.603954068999883, + 10.716208351001114, + 10.828844931998901, + 10.961461735998455, + 11.096193080000376, + 11.21002125599989, + 11.339038371999777, + 11.47282508599892, + 11.578938342998299, + 11.689193653000984, + 11.808900415999233, + 11.939468094999029, + 12.060560442001588, + 12.173040544999822, + 12.29907263000132, + 12.41883604800023, + 12.54484468499868, + 12.681031972999335, + 12.8159874640005, + 12.935219402999792, + 13.052857014001347, + 13.184841923000931, + 13.29681743800029, + 13.40785803900144, + 13.508825118999084, + 13.632424966999679, + 13.75522454699967, + 13.874115894999704, + 13.97497056499924, + 14.076989157998469, + 14.177298460999737, + 14.264653147998615, + 14.352331475001847, + 14.428499328001635, + 14.492602596001234, + 14.54390448200138, + 14.600053063000814, + 14.664585827998962, + 14.719146595998609, + 14.779629923999892 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 125.45374549881672, + "p95": 152.2298009986116, + "max": 449.5644069975242 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 86.97132832529508, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 14.744665195998095, + "end_to_end_tps": 86.81105898202487, + "timing_valid": true, + "steady_tps": 90.8085639435456, + "per_stream_tps": [ + 18.0, + 18.9, + 18.1, + 19.8, + 17.4 + ], + "ttft_s": [ + 0.17, + 0.33, + 0.41, + 0.25, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16550179700061562, + "decode_tps": 18.007311877462755, + "prefill_tps": null, + "itl_mean_ms": 55.53299719607572, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.67109999834793, + "round_gap_p50_ms": 127.15290099731646, + "round_gap_p95_ms": 145.97483400211786, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.32974984099928406, + "decode_tps": 18.880693035976705, + "prefill_tps": null, + "itl_mean_ms": 52.964157517657014, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.99764599834452, + "round_gap_p50_ms": 127.27102100325283, + "round_gap_p95_ms": 145.50219399825437, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.41416075699817156, + "decode_tps": 18.146073697804308, + "prefill_tps": null, + "itl_mean_ms": 55.108340054906805, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.8537669997022, + "round_gap_p50_ms": 125.56106600095518, + "round_gap_p95_ms": 143.5142820009787, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2489542410003196, + "decode_tps": 19.76319092832117, + "prefill_tps": null, + "itl_mean_ms": 50.59911649019055, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.69466300154454, + "round_gap_p50_ms": 127.74369000180741, + "round_gap_p95_ms": 146.49271099915495, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08457592199920327, + "decode_tps": 17.394265665059017, + "prefill_tps": null, + "itl_mean_ms": 57.49021081175991, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.96452499891166, + "round_gap_p50_ms": 125.19229700046708, + "round_gap_p95_ms": 146.4123000005202, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 54.338964414118, + "itl_p50_ms": 0.0, + "itl_p95_ms": 141.8537669997022, + "round_gap_p50_ms": 127.09590300073614, + "round_gap_p95_ms": 145.97483400211786, + "round_gap_mean_ms": 126.42733508759206, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 29898.082946447, + "first": 29898.248448244, + "last": 29912.409362529, + "end": 29912.409479639, + "elapsed_s": 14.32653319200108, + "chunk_offsets_s": [ + 0.16550179700061562, + 0.5379150939988904, + 0.6802089739976509, + 0.8226270829982241, + 0.9542608359988662, + 1.0706781949993456, + 1.1790977050004585, + 1.298306839998986, + 1.4290913519980677, + 1.5452071239997167, + 1.6599874239982455, + 1.7692222459991171, + 1.8834884429998056, + 2.0119122249998327, + 2.145759464998264, + 2.287176655998337, + 2.423281331000908, + 2.5591334809978434, + 2.673663416000636, + 2.7878764259985473, + 2.9183963180003047, + 3.0401516950005316, + 3.158372881000105, + 3.2902163969993126, + 3.4128885949976393, + 3.5352660680000554, + 3.6632918489995063, + 3.779174786999647, + 3.9132865339997807, + 4.0502309650000825, + 4.178025073000754, + 4.299423052998463, + 4.4115119079979195, + 4.525475528000243, + 4.650631739998062, + 4.794439707999118, + 4.924791322999226, + 5.050334184998064, + 5.179886865000299, + 5.2919102249979915, + 5.412611766998452, + 5.531113887998799, + 5.657948611999018, + 5.790595072998258, + 5.928564537000057, + 6.057180443000107, + 6.169171163997817, + 6.299850625000545, + 6.438881227997626, + 6.56809205099853, + 6.688864175001072, + 6.814558630998363, + 6.957502122000733, + 7.104398339997715, + 7.261240167998039, + 7.401410410999233, + 7.530091426000581, + 7.654518867999286, + 7.7877894369994465, + 7.902647582999634, + 8.044808726997871, + 8.19078356099999, + 8.31624777199977, + 8.448908360998757, + 8.584939862997999, + 8.712391983000998, + 8.830187233998004, + 8.962941212997976, + 9.1042181849989, + 9.246483152997826, + 9.367875283998728, + 9.50115146099779, + 9.628328568000143, + 9.747317999997904, + 9.865736673000356, + 10.007407772998704, + 10.146722675999627, + 10.30587567800103, + 10.464013370998146, + 10.585563339998771, + 10.692439267000736, + 10.800048426001013, + 10.932884033998562, + 11.074200718998327, + 11.186110405000363, + 11.314498813000682, + 11.449731394000992, + 11.553085141000338, + 11.662593231998471, + 11.775167935000354, + 11.905239005998737, + 12.035288720999233, + 12.14748399599921, + 12.275456276998739, + 12.392859398998553, + 12.517972721998376, + 12.64948621199801, + 12.776774147998367, + 12.893860755000787, + 13.021013655998104, + 13.153018921999319, + 13.264688880000904, + 13.37841022900102, + 13.478878111000085, + 13.60105976200066, + 13.71812930699889, + 13.83592845900057, + 13.945364430001064, + 14.053473296000448, + 14.152570391001063, + 14.240579052999237, + 14.326416081999923 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 127.15290099731646, + "p95": 145.97483400211786, + "max": 372.41329699827475 + } + }, + { + "start": 29898.08332007, + "first": 29898.413069911, + "last": 29911.918930078, + "end": 29911.919561582, + "elapsed_s": 13.836241511999106, + "chunk_offsets_s": [ + 0.32974984099928406, + 0.5370015939988662, + 0.679478163998283, + 0.8216566880000755, + 0.9537457020014699, + 1.0706284529987897, + 1.17892392900103, + 1.2982022670003062, + 1.428782818999025, + 1.5443833450008242, + 1.6601292550003564, + 1.769165942001564, + 1.883066119000432, + 2.0120738850018824, + 2.1458742159993562, + 2.2874575960013317, + 2.4232438639992324, + 2.559164360001887, + 2.673770327000966, + 2.7877141460012353, + 2.918120105001435, + 3.0394322739994095, + 3.157825487000082, + 3.290755835998425, + 3.4131069659997593, + 3.535995048998302, + 3.663266070001555, + 3.7793055410002125, + 3.9129886480004643, + 4.04961000899857, + 4.177488075001747, + 4.2992891339999915, + 4.411534549999487, + 4.525926761001756, + 4.651300107998395, + 4.793573061000643, + 4.9246790959987266, + 5.050641241999983, + 5.17922954900132, + 5.291822135000984, + 5.412334052001825, + 5.531153936000919, + 5.657798716001707, + 5.789495863999036, + 5.9280703570002515, + 6.056955752999784, + 6.16886635300034, + 6.299540552001417, + 6.438594907998777, + 6.566672950000793, + 6.688595578001696, + 6.813495487000182, + 6.957063731999369, + 7.104470288999437, + 7.260608087999572, + 7.401087573998666, + 7.5295699080015766, + 7.654465235998941, + 7.789430069999071, + 7.901890479999565, + 8.044734895000147, + 8.190237088998401, + 8.315936819999479, + 8.448742042000958, + 8.585045054998773, + 8.712247786999797, + 8.829694427000504, + 8.962958462998358, + 9.103924797000218, + 9.245922442998562, + 9.368368760999147, + 9.500859897001646, + 9.6282356720003, + 9.746750187001453, + 9.865313638998487, + 10.00742948900006, + 10.146639442999003, + 10.305651616999967, + 10.463705011999991, + 10.585259571998904, + 10.691985823999858, + 10.79990848899979, + 10.932463846998871, + 11.074330603001727, + 11.185796383000707, + 11.314216176000627, + 11.449259091001295, + 11.553110010998353, + 11.66180361700026, + 11.77454071000102, + 11.905212803001632, + 12.035508602999471, + 12.147791921001044, + 12.275198290000844, + 12.392667667998467, + 12.5178655790005, + 12.649354515000596, + 12.776450418001332, + 12.893267968000146, + 13.021134108999831, + 13.15355463800006, + 13.264587656998629, + 13.377325348999875, + 13.478785878000053, + 13.600469278000674, + 13.71788535000087, + 13.835610008001822 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 127.3233980009536, + "p95": 145.50219399825437, + "max": 207.2517529995821 + } + }, + { + "start": 29898.084114353, + "first": 29898.49827511, + "last": 29912.550901824, + "end": 29912.550996954, + "elapsed_s": 14.46688260099836, + "chunk_offsets_s": [ + 0.41416075699817156, + 0.5364146159990923, + 0.679258423999272, + 0.8212023539999791, + 0.952713449998555, + 1.0691965709993383, + 1.1784141409989388, + 1.2972937129998172, + 1.4273803039977793, + 1.5442120009975042, + 1.6590910969971446, + 1.7686154059992987, + 1.8819940159992257, + 2.0109660279995296, + 2.1452244369975233, + 2.2874135349993594, + 2.4235087519991794, + 2.558242674000212, + 2.672733719999087, + 2.7870726639994245, + 2.9174515109989443, + 3.0387803159974283, + 3.15689844499866, + 3.2898058889986714, + 3.412109816999873, + 3.5350466379968566, + 3.6623698829971545, + 3.7777779739990365, + 3.912367896999058, + 4.048997318997863, + 4.1771325079971575, + 4.298007732999395, + 4.4102232699988235, + 4.5244032249975135, + 4.649964290998469, + 4.793478572999447, + 4.924305776999972, + 5.049460432997876, + 5.178221872996801, + 5.2911534949998895, + 5.411483897998551, + 5.530147490997479, + 5.657208116997936, + 5.7892063339968445, + 5.926999960000103, + 6.055697841999063, + 6.167701455997303, + 6.299188190998393, + 6.43797668699699, + 6.566344838000077, + 6.6882658029971935, + 6.813115200999164, + 6.956546300996706, + 7.10416100799921, + 7.259988252997573, + 7.399571906997153, + 7.529410839997581, + 7.653523953998956, + 7.786823107999226, + 7.901360460997239, + 8.04348927799947, + 8.190528576997167, + 8.315541645999474, + 8.447814706996724, + 8.584547700000257, + 8.712082682999608, + 8.829513294000208, + 8.962037940997106, + 9.103535654998268, + 9.244666664999386, + 9.366577891996712, + 9.500261145996774, + 9.628030769999896, + 9.74640807600008, + 9.864088137997896, + 10.006800715997088, + 10.145049810998898, + 10.305965034996916, + 10.46229784499883, + 10.584019322999666, + 10.690880100999493, + 10.798669452997274, + 10.932023284000024, + 11.073877050999727, + 11.18537266400017, + 11.314651586999389, + 11.448980695997307, + 11.552554228997906, + 11.661539409997204, + 11.773914385998069, + 11.90436334900005, + 12.034428628998285, + 12.14657889099908, + 12.2746669679982, + 12.392287236998527, + 12.517142830998637, + 12.648653713997192, + 12.775214686000254, + 12.892228061999049, + 13.01959536099821, + 13.151951168998494, + 13.262926201998198, + 13.376774607997504, + 13.478129737999552, + 13.599480992997997, + 13.716857007999351, + 13.835400023999682, + 13.944097026997042, + 14.051772310998786, + 14.15105377999862, + 14.23881582699687, + 14.32495672299774, + 14.40211075499974, + 14.466787470999407 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 125.56106600095518, + "p95": 143.5142820009787, + "max": 160.9152239980176 + } + }, + { + "start": 29898.083742603, + "first": 29898.332696844, + "last": 29911.235471549, + "end": 29911.236370938, + "elapsed_s": 13.152628334999463, + "chunk_offsets_s": [ + 0.2489542410003196, + 0.5369952559994999, + 0.6798423439977341, + 0.8213332539999101, + 0.9533777920005377, + 1.070068729997729, + 1.1780201279980247, + 1.2975524559988116, + 1.427978212999733, + 1.544347478000418, + 1.6595237759975134, + 1.7686481989985623, + 1.8829130359990813, + 2.011017177999747, + 2.1450351370003773, + 2.285903681000491, + 2.422418996997294, + 2.5582770080000046, + 2.672781753997697, + 2.7870332290003716, + 2.917497282000113, + 3.038622869000392, + 3.1580924059999234, + 3.2898382719977235, + 3.411898130998452, + 3.534622601000592, + 3.6625475649998407, + 3.7791346599988174, + 3.9128408639990084, + 4.049636940999335, + 4.176603136998892, + 4.298537163998844, + 4.411059344998648, + 4.524618989998999, + 4.650666757999716, + 4.794027615997038, + 4.924476609998237, + 5.050425560999429, + 5.179124091999256, + 5.291179969997756, + 5.412146426999243, + 5.530244921999838, + 5.657672735997039, + 5.788858608997543, + 5.928079124998476, + 6.05622975499864, + 6.168319344000338, + 6.299173010997038, + 6.437988682999276, + 6.566370734999509, + 6.688125554999715, + 6.812954778997664, + 6.956504548998055, + 7.103276210000331, + 7.259757433999766, + 7.400179719999869, + 7.529236837999633, + 7.653677427999355, + 7.7885056089980935, + 7.901395885997772, + 8.044151733000035, + 8.19064444399919, + 8.315324114999385, + 8.44790742799887, + 8.584391159998631, + 8.711879896000028, + 8.829030436998437, + 8.962051944999985, + 9.103169022997463, + 9.245415596000385, + 9.367775632999837, + 9.50072440799704, + 9.627871282998967, + 9.746171532999142, + 9.864838014997076, + 10.00653267799862, + 10.14575789099763, + 10.30494422799893, + 10.463022721000016, + 10.584708430997125, + 10.691451383998356, + 10.7997257049974, + 10.931935131997307, + 11.074166021000565, + 11.185230333998334, + 11.314482943998883, + 11.448883057997591, + 11.552578273000108, + 11.661601908999728, + 11.774451487999613, + 11.905033731000003, + 12.034535450999101, + 12.146739747997344, + 12.274355378998735, + 12.392414261998056, + 12.517230600999028, + 12.648758839997754, + 12.775826379998762, + 12.892805331997806, + 13.020549021999614, + 13.151728945998912 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 127.83432700052799, + "p95": 146.49271099915495, + "max": 288.04101499918033 + } + }, + { + "start": 29898.082562101, + "first": 29898.167138023, + "last": 29912.82714178, + "end": 29912.827227297, + "elapsed_s": 14.744665195998095, + "chunk_offsets_s": [ + 0.08457592199920327, + 0.5375815139996121, + 0.6801364579987421, + 0.8221755669983395, + 0.9543455799976073, + 1.0706559649988776, + 1.1791232649993617, + 1.2985623189997568, + 1.428664098999434, + 1.545063035999192, + 1.6600557149977249, + 1.7694806849976885, + 1.8837496999985888, + 2.0120870739992824, + 2.146076095999888, + 2.2880406209987996, + 2.4246168339996075, + 2.5600370569991355, + 2.67473723600051, + 2.7886693660002493, + 2.91857803899984, + 3.0397056410001824, + 3.158128093000414, + 3.2909302089974517, + 3.4140482539987715, + 3.536080633999518, + 3.6635989540009177, + 3.7790661929975613, + 3.9138606310007162, + 4.050264089000848, + 4.177565816000424, + 4.299311388000206, + 4.41238671499741, + 4.5255940339993685, + 4.650786330999836, + 4.79444296499787, + 4.925315787997533, + 5.051121772998158, + 5.179481970000779, + 5.292789930997969, + 5.412948606997816, + 5.531653772999562, + 5.658425587000238, + 5.790436964998662, + 5.928624640000635, + 6.056971181998961, + 6.169067072998587, + 6.300454541000363, + 6.439576621000015, + 6.567954417998408, + 6.689561663999484, + 6.8143674729981285, + 6.958144923999498, + 7.105297087000508, + 7.260794088000694, + 7.401178944001003, + 7.531263441000192, + 7.65513388600084, + 7.7889005149991135, + 7.90272599599848, + 8.045127762998163, + 8.191540062998683, + 8.316896959000587, + 8.449403717000678, + 8.586794851999002, + 8.713291872998525, + 8.830813052998565, + 8.963671819998126, + 9.104834805999417, + 9.245948539999517, + 9.368027658998471, + 9.501853467998444, + 9.629345948000264, + 9.74758167300024, + 9.865936025998963, + 10.008097513000394, + 10.146366682998632, + 10.306641621999006, + 10.464570104999439, + 10.585273714998038, + 10.692196284999227, + 10.799748733999877, + 10.933336299000075, + 11.074840409000899, + 11.187040631000855, + 11.315905609000765, + 11.450788835998537, + 11.554297473998304, + 11.663345166998624, + 11.775378418999026, + 11.906115664998651, + 12.036324717999378, + 12.148286128998734, + 12.27627749399835, + 12.393787246001011, + 12.518787028999213, + 12.650260848000471, + 12.776693211999373, + 12.893923576997622, + 13.02110469499894, + 13.153740836998622, + 13.264777024000068, + 13.378583056000934, + 13.479624723997404, + 13.601097774000664, + 13.718726627997967, + 13.836670643999241, + 13.945691332999559, + 14.053485470998567, + 14.152363303997845, + 14.240671465999185, + 14.326692053000443, + 14.403461373000027, + 14.468279896998865, + 14.518563476998679, + 14.575656721000996, + 14.639293928998086, + 14.687422176997643, + 14.74457967899798 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 125.27459650118544, + "p95": 146.4123000005202, + "max": 453.0055920004088 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.6, + "per_stream_tps": 67.6, + "ttft_s_max": 0.1, + "request": { + "ttft_s": 0.095, + "decode_tps": 67.603, + "prefill_tps": 146.783, + "itl_mean_ms": 14.792, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.229, + "round_gap_p50_ms": 73.015, + "round_gap_p95_ms": 82.091, + "tokens_per_chunk": 4.571 + }, + "pooled": { + "aggregate_prefill_tps": 146.783, + "itl_mean_ms": 14.792, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.229, + "round_gap_p50_ms": 73.015, + "round_gap_p95_ms": 82.091, + "tokens_per_chunk": 4.571 + }, + "timing_valid": true, + "end_to_end_tps": 66.14782531201233, + "makespan_s": 3.8701196719994186, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.77097943287585, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8581722439994337, + "end_to_end_tps": 66.35266229965589, + "timing_valid": true, + "per_stream_tps": [ + 67.8 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09537880700008827, + "decode_tps": 67.77097943287585, + "prefill_tps": 146.7831318123642, + "itl_mean_ms": 14.755578396066353, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.93178599947714, + "round_gap_p50_ms": 73.01541199922212, + "round_gap_p95_ms": 82.85562799937907, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.755578396066353, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.93178599947714, + "round_gap_p50_ms": 73.01541199922212, + "round_gap_p95_ms": 82.85562799937907, + "round_gap_mean_ms": 68.4122271090349, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 146.7831318123642 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29927.086825695, + "first": 29927.182204502, + "last": 29930.944876993, + "end": 29930.944997939, + "elapsed_s": 3.8581722439994337, + "chunk_offsets_s": [ + 0.09537880700008827, + 0.14879111299887882, + 0.20278570699883858, + 0.25274201199863455, + 0.3070864929977688, + 0.3759576789998391, + 0.4547178949978843, + 0.5248857179976767, + 0.5979214789986145, + 0.6721127759992669, + 0.745128187998489, + 0.8153327410000202, + 0.891582935997576, + 0.9765595819990267, + 1.0532059309989563, + 1.1300284329990973, + 1.2062244429980638, + 1.2827385059972585, + 1.3555244429990125, + 1.4115246349974768, + 1.4688008009979967, + 1.5279006919990934, + 1.5868926929979352, + 1.6627644099971803, + 1.723617600997386, + 1.7735497719986597, + 1.8332908769989444, + 1.9109877109985973, + 1.9938433389979764, + 2.059964936997858, + 2.1098705189979228, + 2.1726525300000503, + 2.250967556999967, + 2.3279003119969275, + 2.405941085999075, + 2.4859133239988296, + 2.564913541998976, + 2.6393063839968818, + 2.7138053979979304, + 2.7896274799968523, + 2.8658335479995003, + 2.9489082569998573, + 3.0283106659990153, + 3.1038818519991764, + 3.1808136379986536, + 3.2496883039966633, + 3.303476396999031, + 3.3656625439980417, + 3.4438109479997365, + 3.5079316399969684, + 3.5686844119991292, + 3.641744241998822, + 3.7049354779992427, + 3.750513641996804, + 3.801244924998173, + 3.858051297997008 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 73.01541199922212, + "p95": 82.85562799937907, + "max": 84.97664600145072 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.60311132742137, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8719195640005637, + "end_to_end_tps": 66.11707597961937, + "timing_valid": true, + "per_stream_tps": [ + 67.6 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09983085500061861, + "decode_tps": 67.60311132742137, + "prefill_tps": 140.23720421820735, + "itl_mean_ms": 14.792218588234961, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.32816600057413, + "round_gap_p50_ms": 72.08192699908977, + "round_gap_p95_ms": 81.3477369993052, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.792218588234961, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.32816600057413, + "round_gap_p50_ms": 72.08192699908977, + "round_gap_p95_ms": 81.3477369993052, + "round_gap_mean_ms": 68.58210436363483, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 140.23720421820735 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29930.946106016, + "first": 29931.045936871, + "last": 29934.817952611, + "end": 29934.81802558, + "elapsed_s": 3.8719195640005637, + "chunk_offsets_s": [ + 0.09983085500061861, + 0.15618731300128275, + 0.20971380199989653, + 0.260098933998961, + 0.31436357500206213, + 0.3814230720017804, + 0.46048191800218774, + 0.532377292001911, + 0.6044592190010007, + 0.6778470240024035, + 0.7510013999999501, + 0.8198870810010703, + 0.894347731002199, + 0.9726756560012291, + 1.0510038220018032, + 1.133271318001789, + 1.2135259690003295, + 1.292174194000836, + 1.3622623660012323, + 1.4179125570008182, + 1.4755014239999582, + 1.5352699319992098, + 1.5937758959989878, + 1.6697918280005979, + 1.7328705850013648, + 1.7828104950021952, + 1.840420032000111, + 1.9175084210000932, + 1.9969558520024293, + 2.06529938300082, + 2.119862218001799, + 2.183944659002009, + 2.2607385699993756, + 2.338515654002549, + 2.416994359002274, + 2.498342096001579, + 2.578463390000252, + 2.652384492001147, + 2.726778231000935, + 2.8011917610019736, + 2.8759727450014907, + 2.9545412530023896, + 3.0317561100018793, + 3.1141264969992335, + 3.193706755999301, + 3.263484975999745, + 3.317605108000862, + 3.379716778999864, + 3.458701100000326, + 3.5237503870012006, + 3.5842767499998445, + 3.656461153001146, + 3.7194080490007764, + 3.7651037750001706, + 3.8153039930002706, + 3.871846595000534 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 72.08192699908977, + "p95": 81.3477369993052, + "max": 82.37038699735422 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.54554015079344, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8701196719994186, + "end_to_end_tps": 66.14782531201233, + "timing_valid": true, + "per_stream_tps": [ + 67.5 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.0948217309996835, + "decode_tps": 67.54554015079344, + "prefill_tps": 147.64548012782777, + "itl_mean_ms": 14.804826458823621, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.22896500126808, + "round_gap_p50_ms": 73.08372399711516, + "round_gap_p95_ms": 82.09122699918225, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.804826458823621, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.22896500126808, + "round_gap_p50_ms": 73.08372399711516, + "round_gap_p95_ms": 82.09122699918225, + "round_gap_mean_ms": 68.64055903636405, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 147.64548012782777 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29934.818607106, + "first": 29934.913428837, + "last": 29938.688659584, + "end": 29938.688726778, + "elapsed_s": 3.8701196719994186, + "chunk_offsets_s": [ + 0.0948217309996835, + 0.150268656998378, + 0.20678170199971646, + 0.26120759299737983, + 0.31679732100019464, + 0.3852213779973681, + 0.4632434609993652, + 0.5345913089986425, + 0.6060442979978689, + 0.6815718920006475, + 0.7546556159977627, + 0.8252217490007752, + 0.8990425519987184, + 0.9762715169999865, + 1.0530478859982395, + 1.1274828429995978, + 1.209833996999805, + 1.2919252239989873, + 1.3644383949977055, + 1.4188731920003192, + 1.4773598789979587, + 1.5380912500004342, + 1.597546084998612, + 1.6739346819995262, + 1.7369234879988653, + 1.7860423229976732, + 1.8446518149976328, + 1.9198788089997834, + 1.9979487460004748, + 2.0635244699988107, + 2.1137738109973725, + 2.1758996520002256, + 2.2587770049976825, + 2.338866895999672, + 2.4178909469992504, + 2.4982854509980825, + 2.5782645199979015, + 2.6523011789977318, + 2.7255998410000757, + 2.8003896169975633, + 2.8755957549983577, + 2.9545949759994983, + 3.030590205999033, + 3.1056920169976365, + 3.1857607639976777, + 3.2598903219986823, + 3.31549685400023, + 3.3780364739977813, + 3.456533402997593, + 3.521172379998461, + 3.580888544998743, + 3.6548014969994256, + 3.717768440998043, + 3.763635052000609, + 3.814003646999481, + 3.8700524779997068 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 73.08372399711516, + "p95": 82.09122699918225, + "max": 82.87735299745691 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.3, + "per_stream_tps": 63.0, + "ttft_s_max": 0.22, + "request": { + "ttft_s": 0.141, + "decode_tps": 62.935, + "prefill_tps": 110.291, + "itl_mean_ms": 15.891, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.69, + "round_gap_p50_ms": 76.065, + "round_gap_p95_ms": 85.02, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 148.633, + "itl_mean_ms": 15.908, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.676, + "round_gap_p50_ms": 76.21, + "round_gap_p95_ms": 84.961, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 121.97978083497689, + "makespan_s": 4.197416953000356, + "steady_tps": 126.4, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.29487138656141, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.197416953000356, + "end_to_end_tps": 121.97978083497689, + "timing_valid": true, + "steady_tps": 126.16012654770304, + "per_stream_tps": [ + 63.6, + 62.1 + ], + "ttft_s": [ + 0.19, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18630880799901206, + "decode_tps": 63.58977779686479, + "prefill_tps": 75.14405867528409, + "itl_mean_ms": 15.725797992162565, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.56508799927542, + "round_gap_p50_ms": 75.61780600008206, + "round_gap_p95_ms": 85.29952500248328, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09365078800328774, + "decode_tps": 62.147435693280705, + "prefill_tps": 149.49153443864788, + "itl_mean_ms": 16.090768490197235, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.90707800147356, + "round_gap_p50_ms": 75.79146200077957, + "round_gap_p95_ms": 85.46474099784973, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.9082832411799, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.67603200106532, + "round_gap_p50_ms": 75.79146200077957, + "round_gap_p95_ms": 85.3443400010292, + "round_gap_mean_ms": 72.43950404465848, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 150.06299564057508 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29938.689912661, + "first": 29938.876221469, + "last": 29942.886299957, + "end": 29942.887050117, + "elapsed_s": 4.197137455998018, + "chunk_offsets_s": [ + 0.18630880799901206, + 0.24060247499801335, + 0.29682726499959244, + 0.3546232090011472, + 0.4102558869999484, + 0.47957462499834946, + 0.555790276001062, + 0.6334644019989355, + 0.7112514299988106, + 0.7903765579976607, + 0.8670256939985848, + 0.9440545760007808, + 1.023616435999429, + 1.1044871390004118, + 1.1841333859993028, + 1.2646984739985783, + 1.3531561889976729, + 1.4384557140001561, + 1.5125223729992285, + 1.5732930739977746, + 1.6262928510004713, + 1.6876900489987747, + 1.7543809689996124, + 1.8342309910003678, + 1.9135218250012258, + 1.9810068599981605, + 2.034957668998686, + 2.08749095299936, + 2.153486282997619, + 2.2319662499976403, + 2.2995584929994948, + 2.3576687179993314, + 2.4200762000000395, + 2.4956940060001216, + 2.5767288589995587, + 2.6591542520000075, + 2.7443379899996216, + 2.8280604889987444, + 2.9063537010006257, + 2.986744968999119, + 3.06559401299819, + 3.144330324001203, + 3.226578555000742, + 3.308327875998657, + 3.393672215999686, + 3.477041538000776, + 3.54955430499831, + 3.6023720630000753, + 3.6657825549991685, + 3.7471456620005483, + 3.8164948529993126, + 3.873075970001082, + 3.939191925997875, + 4.013142514999345, + 4.074360579998029, + 4.124485198997718, + 4.1963872960004664 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 75.9167285013973, + "p95": 85.29952500248328, + "max": 88.4577149990946 + } + }, + { + "start": 29938.689633164, + "first": 29938.783283952, + "last": 29942.886429917, + "end": 29942.886587987, + "elapsed_s": 4.196954823000851, + "chunk_offsets_s": [ + 0.09365078800328774, + 0.24069438700098544, + 0.2969294170034118, + 0.3550028250028845, + 0.4103226490005909, + 0.47994185900097364, + 0.5557333210017532, + 0.6335085740029172, + 0.7113252160015691, + 0.7904537990034441, + 0.8674970480024058, + 0.9441063790000044, + 1.0236699510023755, + 1.1049665250029648, + 1.1842757890008215, + 1.265182867002295, + 1.3536830110024312, + 1.438622772002418, + 1.5128789040027186, + 1.5735210710008687, + 1.6266745810025895, + 1.6879059510029037, + 1.7545556100012618, + 1.834720600003493, + 1.9139671180018922, + 1.9814315780022298, + 2.0350540120016376, + 2.087538484000106, + 2.1535822410005494, + 2.2319116110011237, + 2.2996850890012865, + 2.357740390001709, + 2.420465993000107, + 2.49616356900151, + 2.5768396010025754, + 2.6592073820029327, + 2.7446721230007824, + 2.828132145001291, + 2.906484969000303, + 2.9869715260028897, + 3.066222925001057, + 3.1445638720033458, + 3.226802184002736, + 3.3084775260031165, + 3.393861063002987, + 3.4776792190023116, + 3.549949986001593, + 3.60298744100146, + 3.6663085150030383, + 3.747349628003576, + 3.8169289470024523, + 3.873548766001477, + 3.9396423550024338, + 4.013613406001241, + 4.074463961002039, + 4.124972202000208, + 4.196796753003582 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.20039649918908, + "p95": 85.46474099784973, + "max": 147.0435989976977 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 123.75065322962325, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.217687996999302, + "end_to_end_tps": 121.39352184520648, + "timing_valid": true, + "steady_tps": 126.43540189752588, + "per_stream_tps": [ + 63.7, + 61.9 + ], + "ttft_s": [ + 0.22, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.2158502509992104, + "decode_tps": 63.72852764443703, + "prefill_tps": 64.85978096013987, + "itl_mean_ms": 15.691559760792492, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.5702219986415, + "round_gap_p50_ms": 76.50922700122464, + "round_gap_p95_ms": 84.59195099931094, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09626052100065863, + "decode_tps": 61.87532661481163, + "prefill_tps": 145.4386476871885, + "itl_mean_ms": 16.16153085098417, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.58971100035706, + "round_gap_p50_ms": 76.74254000085057, + "round_gap_p95_ms": 84.90183799949591, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.92654530588833, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.5702219986415, + "round_gap_p50_ms": 76.74254000085057, + "round_gap_p95_ms": 84.7447749983985, + "round_gap_mean_ms": 72.5226616607415, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 129.65363716174514 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29942.887905679, + "first": 29943.10375593, + "last": 29947.105103669, + "end": 29947.105483923, + "elapsed_s": 4.217578243999014, + "chunk_offsets_s": [ + 0.2158502509992104, + 0.27170547300192993, + 0.3261409619990445, + 0.3784107169994968, + 0.4307553960024961, + 0.49960114500208874, + 0.5763819220010191, + 0.6528911490022438, + 0.7316216889994394, + 0.8088224390012329, + 0.8856543490001059, + 0.9626785690015822, + 1.0424463910021586, + 1.125700548000168, + 1.2084808930012514, + 1.28861221400075, + 1.3691824359993916, + 1.4512155939992226, + 1.5260422400024254, + 1.58654473900242, + 1.6398266300020623, + 1.7013494189995981, + 1.7675377350024064, + 1.845375225002499, + 1.9249669380005798, + 1.9928583680011798, + 2.0463679160020547, + 2.098806355999841, + 2.1693448860023636, + 2.249913559000561, + 2.3164415130013367, + 2.369516046001081, + 2.4279514930021833, + 2.503851516001305, + 2.584963789999165, + 2.667433892002009, + 2.7521786670004076, + 2.8367706179997185, + 2.9175249530017027, + 2.998339082001621, + 3.0775202570002875, + 3.1627739630021097, + 3.2473309050001262, + 3.3265631169997505, + 3.4052998190018116, + 3.486685629999556, + 3.55961360999936, + 3.613743895999505, + 3.6758665350025694, + 3.755672795999999, + 3.8253481140018266, + 3.8825298090014257, + 3.9483600450002996, + 4.021064815002319, + 4.083531677999417, + 4.138650574001076, + 4.217197990001296 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.64500200007751, + "p95": 84.59195099931094, + "max": 85.25370600182214 + } + }, + { + "start": 29942.887795926, + "first": 29942.984056447, + "last": 29947.105246814, + "end": 29947.105313723, + "elapsed_s": 4.217517797002074, + "chunk_offsets_s": [ + 0.09626052100065863, + 0.27160602300136816, + 0.32634296499963966, + 0.3783872940002766, + 0.43079056100032176, + 0.49999123200177564, + 0.5767337720026262, + 0.6532438620015455, + 0.7322900620019936, + 0.8091045969995321, + 0.8860672020018683, + 0.9634678330003226, + 1.0428886200024863, + 1.1255765390014858, + 1.208932386001834, + 1.2890572270007397, + 1.3695303660024365, + 1.450908700000582, + 1.5258625390015368, + 1.5870552190026501, + 1.6401651209998818, + 1.7010700270002417, + 1.767292278000241, + 1.8456971420018817, + 1.9252942620005342, + 1.9928989250001905, + 2.0464160240007914, + 2.098780420001276, + 2.16936350800097, + 2.249953219001327, + 2.316800627999328, + 2.3697417760013195, + 2.4277406239998527, + 2.5041842800019367, + 2.5848776980019466, + 2.667638936000003, + 2.752540773999499, + 2.836953198999254, + 2.918000462999771, + 2.9985485899997, + 3.0774045170001045, + 3.1627713220004807, + 3.2476354320024257, + 3.3268403969996143, + 3.405298681002023, + 3.4865629310006625, + 3.559930568000709, + 3.614073830001871, + 3.676388506002695, + 3.7563087939997786, + 3.8255507739995664, + 3.882922683002107, + 3.948162085001968, + 4.021062461000838, + 4.083574202002637, + 4.13896148600179, + 4.217450888001622 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.7785374991945, + "p95": 84.90183799949591, + "max": 175.34550200070953 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.55838852182681, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.189935222999338, + "end_to_end_tps": 122.19759322042408, + "timing_valid": true, + "steady_tps": 126.44690664965812, + "per_stream_tps": [ + 62.3, + 63.7 + ], + "ttft_s": [ + 0.09, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09492451500045718, + "decode_tps": 62.28109910972317, + "prefill_tps": 147.48561001267768, + "itl_mean_ms": 16.05623558823615, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.15449699835153, + "round_gap_p50_ms": 76.3378610026848, + "round_gap_p95_ms": 85.13523799774703, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18838306899851887, + "decode_tps": 63.73233172469538, + "prefill_tps": 74.31665740677616, + "itl_mean_ms": 15.690623156857042, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.79084299970418, + "round_gap_p50_ms": 75.65745999818319, + "round_gap_p95_ms": 84.90565100146341, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.873429372546596, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.79084299970418, + "round_gap_p50_ms": 76.2095119971491, + "round_gap_p95_ms": 84.96076899973559, + "round_gap_mean_ms": 72.28079446427468, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 148.63331481355232 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29947.106345786, + "first": 29947.201270301, + "last": 29951.295610376, + "end": 29951.295764323, + "elapsed_s": 4.189418537000165, + "chunk_offsets_s": [ + 0.09492451500045718, + 0.2421233269997174, + 0.2984766449990275, + 0.35407027000110247, + 0.4072061910010234, + 0.47505381800147006, + 0.5502476840010786, + 0.6264571959982277, + 0.7031458279998333, + 0.7818400809992454, + 0.8587088640015281, + 0.9413221270006034, + 1.0246627829983481, + 1.1059110349997354, + 1.184506108998903, + 1.2629100710000785, + 1.3454532899995684, + 1.4266471279988764, + 1.501446627000405, + 1.561730624998745, + 1.6127816400003212, + 1.6758330199991178, + 1.743499405998591, + 1.8213562939999974, + 1.902510790998349, + 1.9749012760003097, + 2.0297017230004712, + 2.08211076400039, + 2.1482350549995317, + 2.2260734789997514, + 2.292719749999378, + 2.345976820000942, + 2.404811486998369, + 2.481149348001054, + 2.5617232129989134, + 2.6446001840013196, + 2.7297354219990666, + 2.814696190998802, + 2.8971089260012377, + 2.9832049969991203, + 3.063714379000885, + 3.1427708539995365, + 3.2243432360010047, + 3.3039231799994013, + 3.3833014990013908, + 3.463103082998714, + 3.5351331130004837, + 3.5894473330008623, + 3.6529632929996296, + 3.7331778849984403, + 3.80314501400062, + 3.8599031910016492, + 3.928767028999573, + 4.006564362000063, + 4.067895857999247, + 4.118559868999, + 4.1892645900006755 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.5132465021452, + "p95": 85.13523799774703, + "max": 147.19881199926022 + } + }, + { + "start": 29947.10624363, + "first": 29947.294626699, + "last": 29951.295735604, + "end": 29951.296178853, + "elapsed_s": 4.189935222999338, + "chunk_offsets_s": [ + 0.18838306899851887, + 0.24208659399664612, + 0.298882433999097, + 0.35444705299960333, + 0.4076941359999182, + 0.4754510799975833, + 0.5508119449987134, + 0.6264694049968966, + 0.7033795619972807, + 0.7818924959974538, + 0.8589238789973024, + 0.9415417979980703, + 1.0248352559974592, + 1.1063012419981533, + 1.185052052998799, + 1.2628502499974275, + 1.3457544129996677, + 1.4270068729965715, + 1.5017184229982377, + 1.5615926469981787, + 1.61270889199659, + 1.6756787710000935, + 1.743727636996482, + 1.8216901039995719, + 1.9029318939974473, + 1.9748127839993685, + 2.0300712769967504, + 2.0824556939987815, + 2.1487287629970524, + 2.2265583879998303, + 2.293052455999714, + 2.34630258299876, + 2.40531298699716, + 2.4814607279986376, + 2.5621445079996192, + 2.6449074359989027, + 2.7301360629971896, + 2.815041713998653, + 2.897005057999195, + 2.983202548999543, + 3.0639933919992473, + 3.1431149529998947, + 3.224551227998745, + 3.30430350199822, + 3.3837326189968735, + 3.463392447996739, + 3.535423069999524, + 3.5897124109978904, + 3.6533685099966533, + 3.733431220000057, + 3.803515208997851, + 3.8603186319996894, + 3.9287510129979637, + 4.006425517996831, + 4.068196023999917, + 4.119041373000073, + 4.189491973997065 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 75.90260049983044, + "p95": 84.90565100146341, + "max": 86.19749100034824 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 211.6, + "per_stream_tps": 54.4, + "ttft_s_max": 0.38, + "request": { + "ttft_s": 0.237, + "decode_tps": 54.419, + "prefill_tps": 61.373, + "itl_mean_ms": 18.377, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.284, + "round_gap_p50_ms": 87.266, + "round_gap_p95_ms": 97.192, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 147.045, + "itl_mean_ms": 18.346, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.242, + "round_gap_p50_ms": 87.355, + "round_gap_p95_ms": 97.204, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 208.23291035187626, + "makespan_s": 4.917570417997922, + "steady_tps": 221.8, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 211.79936360166766, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.912234670999169, + "end_to_end_tps": 208.45909623281784, + "timing_valid": true, + "steady_tps": 222.00754542633166, + "per_stream_tps": [ + 55.1, + 54.0, + 56.3, + 53.0 + ], + "ttft_s": [ + 0.29, + 0.19, + 0.38, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.2859762660009437, + "decode_tps": 55.13037392110892, + "prefill_tps": 48.9551115404584, + "itl_mean_ms": 18.13882128626574, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.93930799665395, + "round_gap_p50_ms": 87.17354399777832, + "round_gap_p95_ms": 97.59402599956957, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19141481899714563, + "decode_tps": 54.0214033689934, + "prefill_tps": 73.13958278334118, + "itl_mean_ms": 18.51118145097965, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.65293999831192, + "round_gap_p50_ms": 87.27992499916581, + "round_gap_p95_ms": 97.59432999999262, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38043995599946356, + "decode_tps": 56.286743083637134, + "prefill_tps": 36.79949957732552, + "itl_mean_ms": 17.76617272941318, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.58650199969998, + "round_gap_p50_ms": 87.52177600035793, + "round_gap_p95_ms": 97.09125399967888, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09602898299999651, + "decode_tps": 52.95553499560188, + "prefill_tps": 145.78931862686193, + "itl_mean_ms": 18.883767298036986, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.66121599992039, + "round_gap_p50_ms": 87.30830900094588, + "round_gap_p95_ms": 97.59472899895627, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.32498569117389, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.58650199969998, + "round_gap_p50_ms": 87.52177600035793, + "round_gap_p95_ms": 97.40310199777014, + "round_gap_mean_ms": 83.44413127230966, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 146.90407004469597 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29951.297383981, + "first": 29951.583360247, + "last": 29956.208759675, + "end": 29956.209433648, + "elapsed_s": 4.912049667000247, + "chunk_offsets_s": [ + 0.2859762660009437, + 0.44461161399885896, + 0.5073038170012296, + 0.5674714720007614, + 0.6258070209987636, + 0.7055695499984722, + 0.7980721819985774, + 0.8902413030009484, + 0.9798758280012407, + 1.0686042189990985, + 1.155347037998581, + 1.2429007760001696, + 1.3348593759983487, + 1.4236695889994735, + 1.5121674299989536, + 1.601494966998871, + 1.696493843999633, + 1.7935591460009164, + 1.879085150998435, + 1.9475493610007106, + 2.005213550000917, + 2.0727358799995272, + 2.1460897069991915, + 2.2366659629988135, + 2.3262066149982275, + 2.404946859998745, + 2.4647938560010516, + 2.522664813001029, + 2.5961175480006204, + 2.6832910919983988, + 2.764397414000996, + 2.824279385000409, + 2.8909783529998094, + 2.977095217000169, + 3.0679627409990644, + 3.159115138998459, + 3.2580607199997758, + 3.3543947450016276, + 3.4463340529982815, + 3.5385503260004043, + 3.6265201040005195, + 3.7176722699987295, + 3.815266295998299, + 3.9049022819999664, + 3.993142584000452, + 4.081021725000028, + 4.162051860999782, + 4.220036328999413, + 4.288596097998379, + 4.380760372001532, + 4.461643423001078, + 4.524696570999367, + 4.597265929998684, + 4.680245125000511, + 4.757800532999681, + 4.816924057999131, + 4.911375693998707 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.36364099968341, + "p95": 97.59402599956957, + "max": 158.63534799791523 + } + }, + { + "start": 29951.297436379, + "first": 29951.488851198, + "last": 29956.209202468, + "end": 29956.209331618, + "elapsed_s": 4.911895238998113, + "chunk_offsets_s": [ + 0.19141481899714563, + 0.4446246859988605, + 0.5077330849999271, + 0.5674849600000016, + 0.6256404179985111, + 0.705750569999509, + 0.7978969559990219, + 0.8899776159996691, + 0.9799374589965737, + 1.0683037329981744, + 1.1555014819969074, + 1.2431556009978522, + 1.3345710969988431, + 1.4237905619993398, + 1.512455358999432, + 1.6014090979988396, + 1.6962044599968067, + 1.7936107129971788, + 1.8791593889982323, + 1.9472798649985634, + 2.0051164969991078, + 2.072502334998717, + 2.145836257997871, + 2.237723663998622, + 2.326263333998213, + 2.404928412997833, + 2.464934268999059, + 2.522556336996786, + 2.5959240659976786, + 2.6832039909968444, + 2.764080061999266, + 2.8241178529970057, + 2.8908841319971543, + 2.9772892289984156, + 3.0677533069974743, + 3.1591968499997165, + 3.258096271998511, + 3.3545626459999767, + 3.4463784049985406, + 3.5383931459982705, + 3.626429386997188, + 3.7175208659973578, + 3.8151151959973504, + 3.905233892997785, + 3.9933860359997198, + 4.081066060996818, + 4.162290065996785, + 4.2201197209979, + 4.288749743998778, + 4.38040268399709, + 4.461751885999547, + 4.525828201996774, + 4.5975394149973, + 4.680088903998694, + 4.757817797999451, + 4.817439826998452, + 4.911766088996956 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.4670220000553, + "p95": 97.59432999999262, + "max": 253.20986700171488 + } + }, + { + "start": 29951.298056183, + "first": 29951.678496139, + "last": 29956.208870185, + "end": 29956.20894436, + "elapsed_s": 4.910888177000743, + "chunk_offsets_s": [ + 0.38043995599946356, + 0.4440999189973809, + 0.5071946059979382, + 0.5667310159988119, + 0.6248997539987613, + 0.7050602079980308, + 0.797671331998572, + 0.8887319579989708, + 0.9795638869982213, + 1.0672870609996608, + 1.1548339349974412, + 1.2424199759989278, + 1.3342414760008978, + 1.4231265509988589, + 1.5110261359986907, + 1.6009034989983775, + 1.6957216599985259, + 1.7928129139982047, + 1.8780985730008979, + 1.9477973110006133, + 2.004669231999287, + 2.0723912059984286, + 2.145134327998676, + 2.236720829998376, + 2.325811332997546, + 2.404416494999168, + 2.46434600000066, + 2.522209277998627, + 2.595250934999058, + 2.6827727109994157, + 2.76328779799951, + 2.8240669000006164, + 2.8901419949979754, + 2.9761462210008176, + 3.0673680899999454, + 3.1587837939987367, + 3.2571247960004257, + 3.3534029179973004, + 3.4450248409993947, + 3.537663311999495, + 3.6256631539981754, + 3.71744801100067, + 3.81485111299844, + 3.903997764999076, + 3.99209990899908, + 4.080069767998793, + 4.16193556899816, + 4.219220833998406, + 4.287986437997461, + 4.38015333599833, + 4.460935478000465, + 4.525902002998919, + 4.597155575000215, + 4.679415308997704, + 4.75681387199802, + 4.816548186998261, + 4.9108140019998245 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.53432499906921, + "p95": 97.09125399967888, + "max": 98.34100200168905 + } + }, + { + "start": 29951.297294992, + "first": 29951.393323975, + "last": 29956.208684636, + "end": 29956.209529663, + "elapsed_s": 4.912234670999169, + "chunk_offsets_s": [ + 0.09602898299999651, + 0.44462154999928316, + 0.5071105259994511, + 0.5673723229992902, + 0.6254587590010487, + 0.7057368889982172, + 0.7982356809989142, + 0.8898968969988346, + 0.9796699779981282, + 1.0682490299986966, + 1.1550053840001056, + 1.2428163459990174, + 1.3346690129983472, + 1.423667459999706, + 1.5122092839992547, + 1.6014856379988487, + 1.6962311940005748, + 1.793825922999531, + 1.8791392279999855, + 1.9488702540002123, + 2.0050952159981534, + 2.072988160998648, + 2.146351170998969, + 2.237197804999596, + 2.326182343000255, + 2.4049981700009084, + 2.4653154260013252, + 2.522655869001028, + 2.5958967519982252, + 2.683205060999171, + 2.7646599350009637, + 2.824648687001172, + 2.891124044999742, + 2.9771185589997913, + 3.067980563999299, + 3.159449659000529, + 3.258104493997962, + 3.3542458829979296, + 3.446053497998946, + 3.5388404469995294, + 3.626661652000621, + 3.71803263000038, + 3.8151793840006576, + 3.905277745998319, + 3.9929929219979385, + 4.081032587000664, + 4.1620816989998275, + 4.220085526998446, + 4.288617711998086, + 4.380678019999323, + 4.46165372500036, + 4.52494837300037, + 4.597293623999576, + 4.67999789500027, + 4.7576680530000885, + 4.817264483001054, + 4.911389643999428 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.51174250028271, + "p95": 97.59472899895627, + "max": 348.59256699928665 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 211.58888964545793, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.917570417997922, + "end_to_end_tps": 208.23291035187626, + "timing_valid": true, + "steady_tps": 221.75903834315108, + "per_stream_tps": [ + 55.1, + 56.2, + 54.0, + 52.9 + ], + "ttft_s": [ + 0.29, + 0.38, + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.2852971169995726, + "decode_tps": 55.068035656424094, + "prefill_tps": 49.0716490486687, + "itl_mean_ms": 18.15935484314561, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.34766100053093, + "round_gap_p50_ms": 87.08996200221009, + "round_gap_p95_ms": 97.1803690008528, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3804418620020442, + "decode_tps": 56.2092462878104, + "prefill_tps": 36.79931521291097, + "itl_mean_ms": 17.790667301953505, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.21204600180499, + "round_gap_p50_ms": 87.25240799685707, + "round_gap_p95_ms": 95.67492100177333, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19076391499766032, + "decode_tps": 53.965438388601676, + "prefill_tps": 73.38914175761022, + "itl_mean_ms": 18.530378513726948, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.27708500111476, + "round_gap_p50_ms": 87.56632400036324, + "round_gap_p95_ms": 96.73204899809207, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09678825499941013, + "decode_tps": 52.90410592652384, + "prefill_tps": 144.64564941361246, + "itl_mean_ms": 18.902124560782777, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.35746900230879, + "round_gap_p50_ms": 87.3190109996358, + "round_gap_p95_ms": 97.13792099864804, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.34563130490221, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.24180599994725, + "round_gap_p50_ms": 87.35500399779994, + "round_gap_p95_ms": 96.73204899809207, + "round_gap_mean_ms": 83.53814254910829, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 147.04450213642806 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29956.210773771, + "first": 29956.496070888, + "last": 29961.126706373, + "end": 29961.127179429, + "elapsed_s": 4.9164056580011675, + "chunk_offsets_s": [ + 0.2852971169995726, + 0.4409136380018026, + 0.5044785649988626, + 0.5630518430007214, + 0.6208578920013679, + 0.6997201970007154, + 0.7857613710002624, + 0.8803849180003454, + 0.9737542889997712, + 1.0608442510019813, + 1.1474706500011962, + 1.237361711999256, + 1.3279848769998353, + 1.4188400080020074, + 1.5062718369990762, + 1.5970563030023186, + 1.6914007689993014, + 1.7842559849996178, + 1.8761578980011109, + 1.9442004810007347, + 2.0013223019996076, + 2.0691966459999094, + 2.1411085530016862, + 2.234276917999523, + 2.3248252590019547, + 2.4040433379996102, + 2.464996627000801, + 2.5214633979994687, + 2.591753692999191, + 2.6797051089997694, + 2.756658800000878, + 2.8175306529992667, + 2.8854994220018853, + 2.9731136570007948, + 3.0651560240003164, + 3.1575036850008473, + 3.2554646079988743, + 3.3520672830018157, + 3.4435530619994097, + 3.5363830880014575, + 3.6233580379994237, + 3.7142430349995266, + 3.809357252001064, + 3.9051371310015384, + 3.9945287090013153, + 4.0841900300001726, + 4.165615118999995, + 4.2238655559995095, + 4.294664933000604, + 4.386183911999979, + 4.467398778000643, + 4.531453790001251, + 4.6032163230011065, + 4.68677662600021, + 4.757994782001333, + 4.8187522330008505, + 4.915932602001703 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.26089549963945, + "p95": 97.1803690008528, + "max": 155.61652100223 + } + }, + { + "start": 29956.210683101, + "first": 29956.591124963, + "last": 29961.127745125, + "end": 29961.127858293, + "elapsed_s": 4.917175192000286, + "chunk_offsets_s": [ + 0.3804418620020442, + 0.44120950500291656, + 0.5050696110010904, + 0.5635947930022667, + 0.6214880590014218, + 0.7004076430021087, + 0.786413858000742, + 0.8805844670023362, + 0.9739224300028582, + 1.0611748379997152, + 1.148171408000053, + 1.2378094820014667, + 1.328441126999678, + 1.41938358400148, + 1.5069627390003006, + 1.5976817840019066, + 1.6916109900012088, + 1.7840501140017295, + 1.8759032750022016, + 1.9445505570001842, + 2.0016135780024342, + 2.069574241002556, + 2.141757442001108, + 2.234699057000398, + 2.3246344440012763, + 2.4046198920004827, + 2.465262095000071, + 2.522295677001239, + 2.59236424700066, + 2.680146496000816, + 2.7568044130021008, + 2.8180808720026107, + 2.8856801399997494, + 2.973658308001177, + 3.065870354002982, + 3.158115184000053, + 3.2561013859995, + 3.3517763070012734, + 3.444031889001053, + 3.536006288999488, + 3.6241438739998557, + 3.71418033000009, + 3.8096473140030866, + 3.9050256260015885, + 3.9948361790011404, + 4.084855274002621, + 4.166062587999477, + 4.224610017001396, + 4.295429904999764, + 4.3866415890006465, + 4.467698055002984, + 4.532123338001838, + 4.603935793002165, + 4.687182976002077, + 4.7585757880005985, + 4.819176215001789, + 4.917062024000188 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.4157814978389, + "p95": 95.67492100177333, + "max": 97.98620199944708 + } + }, + { + "start": 29956.210379633, + "first": 29956.401143548, + "last": 29961.126390069, + "end": 29961.126769301, + "elapsed_s": 4.916389668000193, + "chunk_offsets_s": [ + 0.19076391499766032, + 0.4414610539970454, + 0.5056531219997851, + 0.5638093189991196, + 0.6217432720004581, + 0.7005695289990399, + 0.7862479879986495, + 0.8812361870004679, + 0.9743600559995684, + 1.0617240139981732, + 1.148243438998179, + 1.2379015279984742, + 1.328462485998898, + 1.4194335989996034, + 1.5073897099973692, + 1.598051459997805, + 1.6921614799975941, + 1.7851280069990025, + 1.876098696000554, + 1.9450415909996082, + 2.001872662000096, + 2.069830461998208, + 2.1421328609976626, + 2.234975580999162, + 2.3250389349996112, + 2.4049564309971174, + 2.4656102349981666, + 2.5225475929983077, + 2.59276896299707, + 2.680528107997816, + 2.7569379939995997, + 2.818603618998168, + 2.886164822997671, + 2.9742887180000253, + 3.0660753899974225, + 3.1585348429980513, + 3.2567900199974247, + 3.3521716789982747, + 3.444413484998222, + 3.5366905699993367, + 3.6242568939997, + 3.7145366779986944, + 3.8095367189998797, + 3.905468725999526, + 3.9950775670004077, + 4.0852141979994485, + 4.166402631999517, + 4.224604748997081, + 4.295782044999214, + 4.386695329998474, + 4.468068322999898, + 4.532164918997296, + 4.604124092998973, + 4.687530107999919, + 4.7586643760005245, + 4.81927838699994, + 4.916010435998032 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.6627345005545, + "p95": 96.73204899809207, + "max": 250.69713899938506 + } + }, + { + "start": 29956.210287875, + "first": 29956.30707613, + "last": 29961.127117893, + "end": 29961.127474661, + "elapsed_s": 4.917186785998638, + "chunk_offsets_s": [ + 0.09678825499941013, + 0.44115437000073143, + 0.5052874479988532, + 0.5636695440007315, + 0.6216974149974703, + 0.7005316889990354, + 0.7865387199999532, + 0.8814176720006799, + 0.9741728899971349, + 1.0614489929976116, + 1.1476967569979024, + 1.2377506169978005, + 1.3286089799985348, + 1.4193718549977348, + 1.5069663539979956, + 1.5973267929985013, + 1.6917833219995373, + 1.7849944869994943, + 1.8764379079984792, + 1.944558089999191, + 2.0015797359992575, + 2.069349360997876, + 2.1412968519980495, + 2.2345208639999328, + 2.3248998149974796, + 2.404218869000033, + 2.4651429900004587, + 2.521732143999543, + 2.5924597150005866, + 2.679779439000413, + 2.7567867459983972, + 2.8181085159994836, + 2.8858996870003466, + 2.9732546909981465, + 3.0653122259973316, + 3.1576696949996403, + 3.255617689999781, + 3.352377835999505, + 3.443809150998277, + 3.536247051997634, + 3.62356606299727, + 3.7144723559977137, + 3.8096897729992634, + 3.905503587997373, + 3.9953744919985184, + 4.084373526999116, + 4.16581239199877, + 4.224024908999127, + 4.294857549997687, + 4.386960542997258, + 4.467587074999756, + 4.531664006997744, + 4.603447994999442, + 4.686966681998456, + 4.758272325998405, + 4.81969209700037, + 4.916830017999018 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.31936749973102, + "p95": 97.13792099864804, + "max": 344.3661150013213 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 210.7250688909119, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.93543064899859, + "end_to_end_tps": 207.47936154421942, + "timing_valid": true, + "steady_tps": 220.73416876461843, + "per_stream_tps": [ + 54.8, + 56.0, + 52.7, + 53.7 + ], + "ttft_s": [ + 0.28, + 0.38, + 0.09, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.2822247860021889, + "decode_tps": 54.81741406695255, + "prefill_tps": 49.60584858018607, + "itl_mean_ms": 18.242378211759977, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.51885399862658, + "round_gap_p50_ms": 86.92365800015978, + "round_gap_p95_ms": 97.27300899976399, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.37753544299994246, + "decode_tps": 55.963262345402775, + "prefill_tps": 37.082611075543795, + "itl_mean_ms": 17.868865360779797, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.29171500192024, + "round_gap_p50_ms": 86.73357700172346, + "round_gap_p95_ms": 96.77615699911257, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09442022400253336, + "decode_tps": 52.681267222727975, + "prefill_tps": 148.2733190680036, + "itl_mean_ms": 18.982079450977515, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.3903219991189, + "round_gap_p50_ms": 86.88295300089521, + "round_gap_p95_ms": 97.20420799931162, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18810788900009356, + "decode_tps": 53.72598105145292, + "prefill_tps": 74.42537404687496, + "itl_mean_ms": 18.612968631364936, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.30371400190052, + "round_gap_p50_ms": 87.43991600204026, + "round_gap_p95_ms": 97.44278699872666, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.426572913720555, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.30371400190052, + "round_gap_p50_ms": 86.95597800033283, + "round_gap_p95_ms": 97.20420799931162, + "round_gap_mean_ms": 83.9067159464061, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 148.11189168991223 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29961.129714246, + "first": 29961.411939032, + "last": 29966.063745476, + "end": 29966.063995048, + "elapsed_s": 4.9342808020010125, + "chunk_offsets_s": [ + 0.2822247860021889, + 0.4391036619999795, + 0.5014154330019664, + 0.5599568990000989, + 0.6165931970026577, + 0.6928891150018899, + 0.7797779249995074, + 0.8710190679994412, + 0.9672654759997386, + 1.0553264650006895, + 1.1413704560000042, + 1.2291014610018465, + 1.3229077649993997, + 1.4148319820014876, + 1.5017556400016474, + 1.5913866980008606, + 1.686951677002071, + 1.7784450330000254, + 1.8631808110003476, + 1.9363853649992961, + 1.9951813490006316, + 2.062338201001694, + 2.1353200010016735, + 2.22772993900071, + 2.3184187299993937, + 2.39737120800055, + 2.4584658780004247, + 2.515053639999678, + 2.5883854910025548, + 2.6782803269998112, + 2.7540456970018568, + 2.811714629002381, + 2.878157226001349, + 2.972291086000041, + 3.0650865339994198, + 3.158466083001258, + 3.255739092001022, + 3.3526571540023724, + 3.445176008000999, + 3.5365322420002485, + 3.62539439000102, + 3.715381903999514, + 3.8055671590009297, + 3.896479606999492, + 3.991432025002723, + 4.082046709001588, + 4.163254383001913, + 4.220800314000371, + 4.2909687020001, + 4.384874642000796, + 4.467069497000921, + 4.532057086002169, + 4.604146345001936, + 4.688437253000302, + 4.759254797001631, + 4.816398566999851, + 4.934031230000983 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.32733150100103, + "p95": 97.27300899976399, + "max": 156.8788759977906 + } + }, + { + "start": 29961.129615126, + "first": 29961.507150569, + "last": 29966.063711236, + "end": 29966.064488687, + "elapsed_s": 4.934873561000131, + "chunk_offsets_s": [ + 0.37753544299994246, + 0.43929659000059473, + 0.501446632999432, + 0.5601012670013006, + 0.6166441249988566, + 0.6930330670002149, + 0.7797666440019384, + 0.8708204270005808, + 0.9672863400010101, + 1.0552835050002614, + 1.141594441000052, + 1.2290030600015598, + 1.3229485480005678, + 1.4150575029998436, + 1.5020134810001764, + 1.591554939001071, + 1.686998492001294, + 1.7787954340019496, + 1.8633182350022253, + 1.9365299250021053, + 1.9952284200007853, + 2.0623790480021853, + 2.1354697450005915, + 2.228243781999481, + 2.3184662660023605, + 2.397427192001487, + 2.458465941999748, + 2.5151924250021693, + 2.5884381149990077, + 2.6779553790001955, + 2.753998463998869, + 2.811369344999548, + 2.8780126640012895, + 2.97285248299886, + 3.0651441980007803, + 3.1586656360013876, + 3.2554417930005, + 3.3529253480010084, + 3.4455335940001532, + 3.536682034999103, + 3.625936153999646, + 3.7156474260009418, + 3.8056008710009337, + 3.896866555001907, + 3.9916661540009954, + 4.081890321998799, + 4.163153356999828, + 4.220935289999034, + 4.2907807789997605, + 4.38502032299948, + 4.467206345001614, + 4.5320762369992735, + 4.604300938000961, + 4.688580710000679, + 4.759409885999048, + 4.816545400000905, + 4.934096109998791 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 86.84477750102815, + "p95": 96.77615699911257, + "max": 117.55070999788586 + } + }, + { + "start": 29961.12928975, + "first": 29961.223709974, + "last": 29966.064140234, + "end": 29966.064254604, + "elapsed_s": 4.934964854001009, + "chunk_offsets_s": [ + 0.09442022400253336, + 0.43919598200227483, + 0.5011813050005003, + 0.5601055870029086, + 0.6164735010024742, + 0.6930012100019667, + 0.779683570999623, + 0.8713485560001573, + 0.9682541020010831, + 1.0551370550019783, + 1.1413049030015827, + 1.2285731049996684, + 1.3227534590005234, + 1.4147770689996833, + 1.502250264002214, + 1.5912813690010807, + 1.6870812750021287, + 1.778748856002494, + 1.8637045870018483, + 1.9367589960020268, + 1.9954796519996307, + 2.062565127002017, + 2.1355755040021904, + 2.2279658260013093, + 2.318456952001725, + 2.39756343100089, + 2.458636709001439, + 2.51559584899951, + 2.588536113002192, + 2.6780922889993235, + 2.754196767000394, + 2.811516064000898, + 2.8785311140018166, + 2.9728681600026903, + 3.0648745280013827, + 3.1585611200025596, + 3.255643904001772, + 3.352848112001084, + 3.4453700850026507, + 3.5367888170003425, + 3.62569270899985, + 3.71619278900107, + 3.805761285002518, + 3.897333837001497, + 3.99194320999959, + 4.081899390002945, + 4.163158489001944, + 4.22082093300196, + 4.290681269001652, + 4.385118591999344, + 4.467253654001979, + 4.531949702999555, + 4.604352279002342, + 4.689088537001226, + 4.759966865000024, + 4.8165760040028545, + 4.9348504840017995 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.07557749949046, + "p95": 97.20420799931162, + "max": 344.7757579997415 + } + }, + { + "start": 29961.129058038, + "first": 29961.317165927, + "last": 29966.063472928, + "end": 29966.063797941, + "elapsed_s": 4.934739902997535, + "chunk_offsets_s": [ + 0.18810788900009356, + 0.4393461580002622, + 0.5013588250003522, + 0.5605395710008452, + 0.6169227649988898, + 0.6934249869991618, + 0.7801507240001229, + 0.871168810997915, + 0.9681648209989362, + 1.0556354719992669, + 1.1417771909982548, + 1.229217107000295, + 1.3229294419979851, + 1.4150498210001388, + 1.5025447599982726, + 1.591591176998918, + 1.6872701219981536, + 1.7795738360000541, + 1.8641369889992347, + 1.9373048540001037, + 1.9959922609996283, + 2.06270762299755, + 2.135876095999265, + 2.228665460999764, + 2.31890132099943, + 2.3981964249978773, + 2.458755203999317, + 2.515623303999746, + 2.5886436010005127, + 2.6781606709992047, + 2.754615712998202, + 2.811554829997476, + 2.878671561000374, + 2.973550676000741, + 3.065240592997725, + 3.1588797449985577, + 3.256179505999171, + 3.3536222929978976, + 3.4455205489975924, + 3.5369369439977163, + 3.625805474999652, + 3.715587979000702, + 3.8058368029996927, + 3.8973089379978774, + 3.9922963629978767, + 4.082199230997503, + 4.163215078999201, + 4.221147301999736, + 4.290969973997562, + 4.385234478999337, + 4.467587974999333, + 4.532230951997917, + 4.604381859997375, + 4.688860289999866, + 4.7603588519996265, + 4.816859571998066, + 4.934414889998152 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.45528350118548, + "p95": 97.44278699872666, + "max": 251.23826900016866 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 243.0, + "per_stream_tps": 50.4, + "ttft_s_max": 0.49, + "request": { + "ttft_s": 0.282, + "decode_tps": 50.398, + "prefill_tps": 49.574, + "itl_mean_ms": 19.842, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.543, + "round_gap_p50_ms": 92.388, + "round_gap_p95_ms": 106.371, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 148.106, + "itl_mean_ms": 19.841, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.327, + "round_gap_p50_ms": 92.691, + "round_gap_p95_ms": 106.041, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 239.54887319273533, + "makespan_s": 5.343377253000654, + "steady_tps": 257.9, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 243.6890135990717, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.3293342139986635, + "end_to_end_tps": 240.18009541188084, + "timing_valid": true, + "steady_tps": 258.2198424385434, + "per_stream_tps": [ + 52.4, + 51.5, + 48.7, + 50.5, + 49.6 + ], + "ttft_s": [ + 0.46, + 0.37, + 0.1, + 0.28, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.4644724160025362, + "decode_tps": 52.430612312997226, + "prefill_tps": 30.14172535904383, + "itl_mean_ms": 19.072827035287286, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.3005140007881, + "round_gap_p50_ms": 91.65558799941209, + "round_gap_p95_ms": 103.85189999942668, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.372100964999845, + "decode_tps": 51.453416375682416, + "prefill_tps": 37.62419697031915, + "itl_mean_ms": 19.435055443133095, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.65325800294522, + "round_gap_p50_ms": 91.32590200169943, + "round_gap_p95_ms": 104.303637999692, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09672769299868378, + "decode_tps": 48.74582963655445, + "prefill_tps": 144.73621323926855, + "itl_mean_ms": 20.514575450985884, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.54293500049971, + "round_gap_p50_ms": 91.88837300098385, + "round_gap_p95_ms": 104.72438400029205, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.280502132998663, + "decode_tps": 50.51101245521587, + "prefill_tps": 49.91049390724858, + "itl_mean_ms": 19.79766295293766, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.06897299879347, + "round_gap_p50_ms": 92.05113099960727, + "round_gap_p95_ms": 104.02352299934137, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.1889911910002411, + "decode_tps": 49.6164868548341, + "prefill_tps": 74.07752671383578, + "itl_mean_ms": 20.15459101176912, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.91500499745598, + "round_gap_p50_ms": 92.02914699926623, + "round_gap_p95_ms": 104.32906299683964, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.794942378822608, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.32713300077012, + "round_gap_p50_ms": 92.02914699926623, + "round_gap_p95_ms": 103.96612400290905, + "round_gap_mean_ms": 90.13768404642438, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 150.38425701942543 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29966.06654691, + "first": 29966.531019326, + "last": 29971.39459022, + "end": 29971.394660446, + "elapsed_s": 5.328113536001183, + "chunk_offsets_s": [ + 0.4644724160025362, + 0.5308880539996608, + 0.5972336100021494, + 0.657191103000514, + 0.7165301220011315, + 0.8014057890031836, + 0.8922415859997272, + 0.9915687190004974, + 1.0946893280015502, + 1.1884725220006658, + 1.2801281100000779, + 1.3769075980017078, + 1.4774087200021313, + 1.5762037470012729, + 1.6707894200008013, + 1.7670120380025764, + 1.8691327850028756, + 1.9729756800006726, + 2.0682817910019367, + 2.14315180000267, + 2.204002336002304, + 2.2767823340000177, + 2.3550309730017034, + 2.452807870002289, + 2.549780063000071, + 2.6364776290029113, + 2.7003613150009187, + 2.759402483003214, + 2.8365154060011264, + 2.9281959010004357, + 3.018550110002252, + 3.0795774560028804, + 3.1463590820021636, + 3.2379027740025776, + 3.3349176020019513, + 3.4342181160027394, + 3.5398114010022255, + 3.643663301001652, + 3.7437189370029955, + 3.845234337000875, + 3.9436225030003698, + 4.046582724000473, + 4.145961555001122, + 4.2425966520022484, + 4.338756535002176, + 4.436304247003136, + 4.523561739002616, + 4.5848082800002885, + 4.660565040001529, + 4.758627587001683, + 4.849852496001404, + 4.918994771000143, + 5.00212895200093, + 5.091429503001564, + 5.165897259001213, + 5.2238020110016805, + 5.328043310000794 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 91.6680414993607, + "p95": 103.85189999942668, + "max": 105.59328499948606 + } + }, + { + "start": 29966.066013958, + "first": 29966.438114923, + "last": 29971.394054061, + "end": 29971.394116063, + "elapsed_s": 5.328102104998834, + "chunk_offsets_s": [ + 0.372100964999845, + 0.5312025230014115, + 0.5978342589987733, + 0.6579475620019366, + 0.7171548520018405, + 0.8022247299995797, + 0.8928874519988312, + 0.9928128520004975, + 1.0954058349998377, + 1.188942512999347, + 1.2802684150010464, + 1.3771194239998295, + 1.4783384149995982, + 1.5763741970004048, + 1.6715055429995118, + 1.7677250570013712, + 1.8699407349995454, + 1.9733616369994706, + 2.0687426770018646, + 2.144467199999781, + 2.204733196002053, + 2.2769301580010506, + 2.3558169540010567, + 2.4529726389991993, + 2.550928003998706, + 2.637404957000399, + 2.700812857001438, + 2.7599823480013583, + 2.8374986560011166, + 2.9284259019987076, + 3.0191482310001447, + 3.0801576420017227, + 3.147058038000978, + 3.237977523000154, + 3.3352403409990075, + 3.4348935990019527, + 3.5397884840022016, + 3.6440921220018936, + 3.744297330002155, + 3.845820938000543, + 3.943718547001481, + 4.04716631699921, + 4.146663232000719, + 4.242867485001625, + 4.339693961999728, + 4.4366910200005805, + 4.524434220002149, + 4.585925488001521, + 4.660889122998924, + 4.759947041999112, + 4.850298854002176, + 4.919651726999291, + 5.002151265998691, + 5.0924156040018715, + 5.166342240001541, + 5.224109451999539, + 5.328040102998784 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.43129000060435, + "p95": 104.303637999692, + "max": 159.10155800156645 + } + }, + { + "start": 29966.065784227, + "first": 29966.16251192, + "last": 29971.39372866, + "end": 29971.393805494, + "elapsed_s": 5.328021266999713, + "chunk_offsets_s": [ + 0.09672769299868378, + 0.5310938170005102, + 0.598098487000243, + 0.6580882359994575, + 0.7172323399972811, + 0.8024134199986293, + 0.892865002999315, + 0.9931961379988934, + 1.0946854529975099, + 1.1886138499976369, + 1.2803657919976104, + 1.3772953459993005, + 1.478636818999803, + 1.5765241979970597, + 1.6716415759983647, + 1.7683762209999259, + 1.8703411249989585, + 1.973829164999188, + 2.068928631997551, + 2.1445804969989695, + 2.204819659997156, + 2.276979340997059, + 2.3555956989985134, + 2.453266483000334, + 2.5504586009992636, + 2.6376696639999864, + 2.7013682590004464, + 2.760262719999446, + 2.8370029459983925, + 2.9288913189993764, + 3.0188706219996675, + 3.0798246230006043, + 3.1468964790001337, + 3.2379368000001705, + 3.335231314998964, + 3.434774249999464, + 3.539951236998604, + 3.644675620998896, + 3.744244205998257, + 3.8461251669978083, + 3.943858755999827, + 4.047467169999436, + 4.14653629699751, + 4.243017998000141, + 4.340077120999922, + 4.436736617997667, + 4.524696591997781, + 4.586115792997589, + 4.660896671997762, + 4.759705254997243, + 4.850169310997444, + 4.919653658998868, + 5.002500200000213, + 5.092402335998486, + 5.166538481997122, + 5.22416436200001, + 5.327944433000084 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.90838500055543, + "p95": 104.72438400029205, + "max": 434.3661240018264 + } + }, + { + "start": 29966.065649265, + "first": 29966.346151398, + "last": 29971.394555451, + "end": 29971.394879285, + "elapsed_s": 5.32923001999734, + "chunk_offsets_s": [ + 0.280502132998663, + 0.5319640699999582, + 0.5980179899997893, + 0.6582667029979348, + 0.717220931997872, + 0.8024762850000116, + 0.8938659159975941, + 0.9927462569976342, + 1.0954375619985512, + 1.1889459039994108, + 1.280997034999018, + 1.3783230599983654, + 1.4786465629986196, + 1.5772072979998484, + 1.6718176590002258, + 1.7686282100003154, + 1.8699655649979832, + 1.9739890879973245, + 2.0689009979978437, + 2.1441704239987303, + 2.204757225998037, + 2.277578616998653, + 2.3555145769969386, + 2.453750491997198, + 2.5507253569994646, + 2.6372299590002513, + 2.7017164249991765, + 2.7602553429969703, + 2.837474123996799, + 2.928898197998933, + 3.0192276380003022, + 3.0802245359991502, + 3.1471755569982633, + 3.238624270998116, + 3.3359984349990555, + 3.435067407997849, + 3.5410546219973185, + 3.64461344299707, + 3.7444558899987896, + 3.846064107998245, + 3.944183002997306, + 4.04765562099783, + 4.146757790000265, + 4.243441734997759, + 4.339847641997039, + 4.4366084209978, + 4.524587914998847, + 4.585664803998952, + 4.661418427996978, + 4.759588834000169, + 4.850715835997107, + 4.919849694997538, + 5.002908513997681, + 5.0922579299985955, + 5.16649347099883, + 5.224970207997103, + 5.328906185997766 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.77973650023341, + "p95": 104.02352299934137, + "max": 251.46193700129515 + } + }, + { + "start": 29966.065545071, + "first": 29966.254536262, + "last": 29971.39395697, + "end": 29971.394268803, + "elapsed_s": 5.328723732000071, + "chunk_offsets_s": [ + 0.1889911910002411, + 0.5312338670009922, + 0.597860292000405, + 0.6582420630002161, + 0.7176787149983284, + 0.8022253050003201, + 0.8931937919987831, + 0.9932240739981353, + 1.0951045809997595, + 1.188912319001247, + 1.2809414660005132, + 1.3777708259985957, + 1.4785417769999185, + 1.5773989980007173, + 1.6721444809991226, + 1.7682790819999354, + 1.8701947369991103, + 1.9739150540008268, + 2.068921735000913, + 2.1447440990014, + 2.205093904998648, + 2.2778571659982845, + 2.356522837999364, + 2.453683739000553, + 2.55113603800055, + 2.637734448999254, + 2.7015173489999142, + 2.76043817900063, + 2.83830457500153, + 2.929333358999429, + 3.019183796001016, + 3.0806948819990794, + 3.1476481429999694, + 3.238213157001155, + 3.335406564998266, + 3.4349186520012154, + 3.540301132001332, + 3.6446301949981716, + 3.744638773998304, + 3.8463967719981156, + 3.944492801998422, + 4.047433151001314, + 4.14734815599877, + 4.243417125999258, + 4.340098142998613, + 4.436886859999504, + 4.524974324998766, + 4.586429334998684, + 4.661299560000771, + 4.760257699999784, + 4.850558727001044, + 4.920182247999037, + 5.0034051190014, + 5.09284719399875, + 5.166985692001617, + 5.224445774998458, + 5.328411899001367 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.91844250037684, + "p95": 104.32906299683964, + "max": 342.2426760007511 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 242.98649688086266, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.343377253000654, + "end_to_end_tps": 239.54887319273533, + "timing_valid": true, + "steady_tps": 257.85902942859383, + "per_stream_tps": [ + 50.4, + 51.3, + 49.5, + 52.4, + 48.6 + ], + "ttft_s": [ + 0.28, + 0.38, + 0.19, + 0.47, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.28240765200098394, + "decode_tps": 50.398177401718115, + "prefill_tps": 49.57372755590639, + "itl_mean_ms": 19.84198738039898, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.56465600023512, + "round_gap_p50_ms": 93.7172419980925, + "round_gap_p95_ms": 106.37121600302635, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.37660334999964107, + "decode_tps": 51.34929199491958, + "prefill_tps": 37.17439050930732, + "itl_mean_ms": 19.47446520000584, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.42649100028211, + "round_gap_p50_ms": 93.90260800137185, + "round_gap_p95_ms": 106.52218199902563, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18821143999957712, + "decode_tps": 49.47387118397573, + "prefill_tps": 74.38442636659842, + "itl_mean_ms": 20.212689568628168, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.17969900177559, + "round_gap_p50_ms": 93.77084300285787, + "round_gap_p95_ms": 106.4032819995191, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.4724011159996735, + "decode_tps": 52.35548146758676, + "prefill_tps": 29.6358317663451, + "itl_mean_ms": 19.100196807837573, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.96775300128502, + "round_gap_p50_ms": 93.80657500150846, + "round_gap_p95_ms": 104.52341600102955, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09597816600216902, + "decode_tps": 48.60690212934314, + "prefill_tps": 145.86650884414288, + "itl_mean_ms": 20.57320989803046, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.01798400096595, + "round_gap_p50_ms": 93.58686899940949, + "round_gap_p95_ms": 106.51045400300063, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.840509770980205, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.17969900177559, + "round_gap_p50_ms": 93.79226799865137, + "round_gap_p95_ms": 106.04121900178143, + "round_gap_mean_ms": 90.34517842142772, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 148.10641536465428 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29971.395795791, + "first": 29971.678203443, + "last": 29976.737910225, + "end": 29976.738655344, + "elapsed_s": 5.342859553002199, + "chunk_offsets_s": [ + 0.28240765200098394, + 0.5394355860007636, + 0.607828578002227, + 0.6736764350025624, + 0.7362761090007552, + 0.8214697090006666, + 0.910729931001697, + 1.004011288001493, + 1.103072202000476, + 1.1973562780003704, + 1.2898272090023966, + 1.3873165380027785, + 1.488077162000991, + 1.5840446580004937, + 1.6857537569994747, + 1.7843184129997098, + 1.8865409220015863, + 1.9834599220012024, + 2.0762439820027794, + 2.1501564900027006, + 2.2094231250011944, + 2.282806174000143, + 2.3610141400022258, + 2.457536527002958, + 2.555293258999882, + 2.6493108960021345, + 2.716871390999586, + 2.7773887180010206, + 2.8536144730023807, + 2.947331715000473, + 3.028405102999386, + 3.0896079430021928, + 3.1575883730001806, + 3.2518680989996938, + 3.3491502220022085, + 3.448372910999751, + 3.5547441270027775, + 3.665112410999427, + 3.770064740001544, + 3.8712732910025807, + 3.9663096020012745, + 4.062774656002148, + 4.160318215999723, + 4.257606998002302, + 4.354077940999559, + 4.450334651999583, + 4.538286647002678, + 4.599243763001141, + 4.681617239002662, + 4.782749107002019, + 4.873091856999963, + 4.940945179001574, + 5.017892158000905, + 5.107106381001358, + 5.180673476002994, + 5.239708993001841, + 5.342114434002724 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.86743950017262, + "p95": 106.37121600302635, + "max": 257.02793399977963 + } + }, + { + "start": 29971.396029174, + "first": 29971.772632524, + "last": 29976.73862115, + "end": 29976.738966829, + "elapsed_s": 5.342937655001151, + "chunk_offsets_s": [ + 0.37660334999964107, + 0.5389071059980779, + 0.6075458179984707, + 0.6733200560011028, + 0.7357721259977552, + 0.8210871849987598, + 0.9100826789981511, + 1.003985286999523, + 1.102891364000243, + 1.1975859650010534, + 1.2895198559999699, + 1.387126707999414, + 1.4877635209995788, + 1.5837584260007134, + 1.6853219199983869, + 1.783748410998669, + 1.885875460000534, + 1.98308390999955, + 2.0762241419979546, + 2.149875760998839, + 2.209288177000417, + 2.2824910290000844, + 2.3614343629997165, + 2.4572168049999163, + 2.5548771819994727, + 2.6488280480007234, + 2.7165227240002423, + 2.776742583999294, + 2.8528509909992863, + 2.94632297499993, + 3.0284291530006158, + 3.0894968200009316, + 3.156883085001027, + 3.250838975000079, + 3.3484635109998635, + 3.4480466450004315, + 3.554568826999457, + 3.664511766000942, + 3.7701772910004365, + 3.870868349000375, + 3.966257394000422, + 4.062251629999082, + 4.159654671999306, + 4.2575685669980885, + 4.353895183998247, + 4.450244858999213, + 4.537811173999216, + 4.5991189449996455, + 4.681586824000988, + 4.78263172899824, + 4.873438353999518, + 4.940763734000939, + 5.017744746997778, + 5.106006737998541, + 5.1807896430000255, + 5.23957428599897, + 5.342591976001131 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.92673700131127, + "p95": 106.52218199902563, + "max": 162.30375599843683 + } + }, + { + "start": 29971.395855489, + "first": 29971.584066929, + "last": 29976.738302769, + "end": 29976.738753604, + "elapsed_s": 5.342898114999116, + "chunk_offsets_s": [ + 0.18821143999957712, + 0.5392571479969774, + 0.6078627069982758, + 0.6733649689995218, + 0.7360641349987418, + 0.8215722239983734, + 0.910433170000033, + 1.0039884229991003, + 1.1032540630003496, + 1.1970616609978606, + 1.2898187129976577, + 1.3874031159975857, + 1.4884049180000147, + 1.583758904998831, + 1.6855400539971015, + 1.783719752998877, + 1.8860919469989312, + 1.9828731499983405, + 2.0756635299985646, + 2.150494997000351, + 2.209762383998168, + 2.28226765999716, + 2.3611415039995336, + 2.457516989998112, + 2.555185031997098, + 2.648955874999956, + 2.716972130998329, + 2.777080866999313, + 2.853185066000151, + 2.9469773339988024, + 3.0283757260003767, + 3.0897524439969857, + 3.1574462079988734, + 3.2514277629998105, + 3.3487613439974666, + 3.448157942999387, + 3.5545612249989063, + 3.6651329719970818, + 3.7713343579998764, + 3.8716864599991823, + 3.966486473000259, + 4.0626834529975895, + 4.159949177999806, + 4.257622294997418, + 4.3541355909983395, + 4.45021203899887, + 4.538362713999959, + 4.599135679000028, + 4.681444191999617, + 4.7822367349981505, + 4.873844960999122, + 4.940981068997644, + 5.0180391569992935, + 5.106558374998713, + 5.1807159739983035, + 5.239699153000402, + 5.34244727999976 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.78155550075462, + "p95": 106.4032819995191, + "max": 351.0457079974003 + } + }, + { + "start": 29971.395951732, + "first": 29971.868352848, + "last": 29976.738903034, + "end": 29976.739096962, + "elapsed_s": 5.343145229999209, + "chunk_offsets_s": [ + 0.4724011159996735, + 0.5391169200011063, + 0.6071739649996744, + 0.6734732119984983, + 0.7360763270007737, + 0.8212225809984375, + 0.9105361170004471, + 1.0041132110018225, + 1.103014008000173, + 1.197509515001002, + 1.2896509160018468, + 1.3874725909990957, + 1.488068361999467, + 1.5842304409998178, + 1.6851289039987023, + 1.7837721549985872, + 1.8863389789985376, + 1.982858925999608, + 2.0766655010011164, + 2.1501914510008646, + 2.2091841729998123, + 2.282404145000328, + 2.3614641390013276, + 2.4573270639994007, + 2.5549027980014216, + 2.6488098059999174, + 2.716554771999654, + 2.777541155999643, + 2.853251676999207, + 2.947134684000048, + 3.028016848998959, + 3.0895130440003413, + 3.15710336399934, + 3.2515518000000156, + 3.348491015000036, + 3.448293372999615, + 3.5543345920013962, + 3.6651347650004027, + 3.7696581810014322, + 3.870805578000727, + 3.965878930001054, + 4.06208930199864, + 4.160057054999925, + 4.257601255001646, + 4.353691750999133, + 4.450422769001307, + 4.5383705720014405, + 4.599248597001861, + 4.6818320650017995, + 4.782560540999839, + 4.873964487000194, + 4.940643056001136, + 5.017591602998436, + 5.106414466001297, + 5.180587137998373, + 5.239626431000943, + 5.342951301998255 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.8447910011746, + "p95": 104.52341600102955, + "max": 110.80017299900646 + } + }, + { + "start": 29971.395719709, + "first": 29971.491697875, + "last": 29976.737866399, + "end": 29976.73837386, + "elapsed_s": 5.342654151001625, + "chunk_offsets_s": [ + 0.09597816600216902, + 0.5390440210030647, + 0.6076002990012057, + 0.6737985979998484, + 0.7358401750025223, + 0.8210761120026291, + 0.9101510810032778, + 1.0038574270001845, + 1.1030340240031364, + 1.1973174600025231, + 1.2897074450011132, + 1.3873418180010049, + 1.4883455390008749, + 1.5845275060019048, + 1.6854195369996887, + 1.7844375210006547, + 1.8861236430020654, + 1.9829320640019432, + 2.075432241999806, + 2.1500477500012494, + 2.2093067040004826, + 2.282203737002419, + 2.360836133000703, + 2.4578466970015143, + 2.5552288240032794, + 2.648815693002689, + 2.7169957790029002, + 2.7775218420028978, + 2.8531042140020872, + 2.94645054600187, + 3.02844857600212, + 3.0895204350017593, + 3.15705427300054, + 3.2510597970031085, + 3.3486713240017707, + 3.4482449850002013, + 3.554755439003202, + 3.6649042740027653, + 3.770927957000822, + 3.8715274920032243, + 3.9668192050012294, + 4.063046409002709, + 4.160047213001235, + 4.257496721002099, + 4.354400928001269, + 4.450205414002994, + 4.538041067000449, + 4.599160383000708, + 4.681444015001034, + 4.782559786002821, + 4.873067343000002, + 4.9406646540010115, + 5.017767192002793, + 5.106975078000687, + 5.181013649002125, + 5.239574554001592, + 5.342146689999936 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.64660749815812, + "p95": 106.51045400300063, + "max": 443.0658550008957 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 241.70438682101326, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.3711367119976785, + "end_to_end_tps": 238.3108210857533, + "timing_valid": true, + "steady_tps": 257.4215014779622, + "per_stream_tps": [ + 51.3, + 52.3, + 49.2, + 50.2, + 48.4 + ], + "ttft_s": [ + 0.4, + 0.49, + 0.19, + 0.29, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.39721739499873365, + "decode_tps": 51.269147139284456, + "prefill_tps": 35.245183560112295, + "itl_mean_ms": 19.504908035299856, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.51224500057288, + "round_gap_p50_ms": 92.6555769983679, + "round_gap_p95_ms": 106.73872500046855, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.491466424002283, + "decode_tps": 52.271703298942626, + "prefill_tps": 28.486177928474245, + "itl_mean_ms": 19.130809537255473, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.17090300243581, + "round_gap_p50_ms": 92.03668799818843, + "round_gap_p95_ms": 106.20981900137849, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.1897669110003335, + "decode_tps": 49.23148381469963, + "prefill_tps": 73.77471618313582, + "itl_mean_ms": 20.312205168624597, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.71724600129528, + "round_gap_p50_ms": 92.14723100012634, + "round_gap_p95_ms": 106.44507800316205, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2877417490017251, + "decode_tps": 50.16871364595308, + "prefill_tps": 48.65474005274106, + "itl_mean_ms": 19.932741490187006, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.9749600016221, + "round_gap_p50_ms": 92.6905549968069, + "round_gap_p95_ms": 106.84628300077748, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09602937700037728, + "decode_tps": 48.351285738491306, + "prefill_tps": 145.7887204656654, + "itl_mean_ms": 20.68197328626411, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.9584840030584, + "round_gap_p50_ms": 92.38783500040881, + "round_gap_p95_ms": 106.76653499831446, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.912527503526206, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.59898899999098, + "round_gap_p50_ms": 92.6905549968069, + "round_gap_p95_ms": 106.20981900137849, + "round_gap_mean_ms": 90.6731163106997, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 142.2617478287896 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29976.74050414, + "first": 29977.137721535, + "last": 29982.111473084, + "end": 29982.111541696, + "elapsed_s": 5.371037555996736, + "chunk_offsets_s": [ + 0.39721739499873365, + 0.5584596310000052, + 0.625078343997302, + 0.6853733379975893, + 0.7453770089996397, + 0.8311291669997445, + 0.9206120659982844, + 1.0142896919969644, + 1.1152753699971072, + 1.20999326799938, + 1.307838124997943, + 1.4085189379984513, + 1.5090311829990242, + 1.6063085749992752, + 1.7015087660001882, + 1.7980811649977113, + 1.899363557997276, + 1.9962072659982368, + 2.088131563999923, + 2.1639069550001295, + 2.223280957998213, + 2.301168947997212, + 2.382389611997496, + 2.478422642998339, + 2.574900047999108, + 2.6621565249988635, + 2.72742690999803, + 2.787082158996782, + 2.8632512809999753, + 2.9548050869998406, + 3.0364380269966205, + 3.096398476998729, + 3.1651114919986867, + 3.2577670689970546, + 3.3628617099966505, + 3.4628999669985205, + 3.569638691998989, + 3.675403702996846, + 3.777292134000163, + 3.87859397399734, + 3.974338539999735, + 4.070646645999659, + 4.168398191999586, + 4.2649846200001775, + 4.367095020999841, + 4.463235376999364, + 4.551199858997279, + 4.612123911996605, + 4.6880650040002365, + 4.788589288997173, + 4.880299409996951, + 4.949418426000193, + 5.0257498739993025, + 5.114335790996847, + 5.187882902999263, + 5.2463562689990795, + 5.370968944000197 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.16660149852396, + "p95": 106.73872500046855, + "max": 161.24223600127152 + } + }, + { + "start": 29976.740989312, + "first": 29977.232455736, + "last": 29982.110812168, + "end": 29982.110928382, + "elapsed_s": 5.369939070002147, + "chunk_offsets_s": [ + 0.491466424002283, + 0.558885475002171, + 0.6249388190008176, + 0.685062894001021, + 0.7451419120006904, + 0.8303121410026506, + 0.9194392160025018, + 1.0133706289998372, + 1.1152964460015937, + 1.2091150050000579, + 1.3069410940006492, + 1.4082544960001542, + 1.5089067640001304, + 1.6067591900027764, + 1.7012151390008512, + 1.7979324890002317, + 1.8989144200022565, + 1.9959305830016092, + 2.0879672709997976, + 2.1637136370009102, + 2.2227042140002595, + 2.3013770900033705, + 2.3815521670003363, + 2.4774852250011463, + 2.5738778250015457, + 2.6614499180032, + 2.726897528002155, + 2.7865564890016685, + 2.862859474000288, + 2.9543792300028144, + 3.035678377000295, + 3.0959635480030556, + 3.1643742110027233, + 3.2573287630002596, + 3.3624292130007234, + 3.462600116003159, + 3.5690071120006905, + 3.675216931002069, + 3.776387676000013, + 3.878063040003326, + 3.973806678000983, + 4.06923651300167, + 4.167348270002549, + 4.265112791999854, + 4.366851110000425, + 4.462808705000498, + 4.550918890003231, + 4.611990503002744, + 4.687230821000412, + 4.788026753001759, + 4.879752250002639, + 4.949034716002643, + 5.025639570001658, + 5.113580540000839, + 5.186848517001636, + 5.2459754060000705, + 5.369822856002429 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.49561999786238, + "p95": 106.20981900137849, + "max": 123.84745000235853 + } + }, + { + "start": 29976.740866555, + "first": 29976.930633466, + "last": 29982.110245784, + "end": 29982.110557418, + "elapsed_s": 5.369690863000869, + "chunk_offsets_s": [ + 0.1897669110003335, + 0.5585441000002902, + 0.625120569999126, + 0.6852411740001116, + 0.7453942899992398, + 0.8305061329992895, + 0.920356503000221, + 1.0138330170011614, + 1.115207481001562, + 1.2093520560010802, + 1.307221027000196, + 1.4085761429996637, + 1.509293389000959, + 1.6062491180018696, + 1.7018025100005616, + 1.7982913849991746, + 1.8995709939990775, + 1.996405133002554, + 2.0881450070010033, + 2.1640442440002516, + 2.223148905999551, + 2.3009761089997482, + 2.38214283400157, + 2.478273005999654, + 2.5742484030015476, + 2.6621882250001363, + 2.727619114000845, + 2.787420849999762, + 2.8632171060016844, + 2.955199875999824, + 3.036880834999465, + 3.096717313001136, + 3.1650462280013016, + 3.257193459001428, + 3.3627213389991084, + 3.4633203279990994, + 3.5697654060022614, + 3.6755160489992704, + 3.777514853001776, + 3.8783455650009273, + 3.974691809999058, + 4.0697258330001205, + 4.167986782002117, + 4.265443992000655, + 4.36700626899983, + 4.463464674001443, + 4.551106066999637, + 4.611110182999255, + 4.686927811002533, + 4.788018286999431, + 4.880147886000486, + 4.9494001580023905, + 5.025706436001201, + 5.113621563999914, + 5.187252265001007, + 5.245844949000457, + 5.369379228999605 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.81187250053335, + "p95": 106.44507800316205, + "max": 368.7771889999567 + } + }, + { + "start": 29976.74063837, + "first": 29977.028380119, + "last": 29982.111229199, + "end": 29982.111322676, + "elapsed_s": 5.370684305999021, + "chunk_offsets_s": [ + 0.2877417490017251, + 0.5589363239996601, + 0.6251921080001921, + 0.6853499930002727, + 0.7459396890008065, + 0.8305490620005003, + 0.9199224350013537, + 1.013945661001344, + 1.1153929440006323, + 1.209361399000045, + 1.3078431020003336, + 1.4089633590010635, + 1.5101218419986253, + 1.6074940860016795, + 1.7016914949999773, + 1.7985861650013248, + 1.8995349739998346, + 1.9968171340005938, + 2.0886493010002596, + 2.1642324269996607, + 2.222966367000481, + 2.3011158900008013, + 2.3821803540013207, + 2.478669470998284, + 2.5745428150003136, + 2.6626641180009756, + 2.7276662980002584, + 2.7874784500017995, + 2.8640272070006176, + 2.9551347990018257, + 3.036617572000978, + 3.0968235719992663, + 3.1650967720015615, + 3.2577873269983684, + 3.362489867999102, + 3.463464828000724, + 3.5703111110015016, + 3.6757863800012274, + 3.777590837999014, + 3.878605048001191, + 3.9746731339982944, + 4.070819439999468, + 4.168137586999364, + 4.265024366999569, + 4.366880947000027, + 4.4635962460015435, + 4.551480851001543, + 4.611406371001067, + 4.687199487001635, + 4.788181012001587, + 4.880273153001326, + 4.949710475000757, + 5.0257329779997235, + 5.114305551000143, + 5.187540484999772, + 5.2461330740006815, + 5.370590828999411 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.32950499810977, + "p95": 106.84628300077748, + "max": 271.194574997935 + } + }, + { + "start": 29976.740404984, + "first": 29976.836434361, + "last": 29982.110337549, + "end": 29982.110411282, + "elapsed_s": 5.370006297998771, + "chunk_offsets_s": [ + 0.09602937700037728, + 0.5583874839976488, + 0.6249720349987911, + 0.6853017350003938, + 0.7455373349985166, + 0.8304654259991366, + 0.9203285330004292, + 1.0138287909976498, + 1.1153105390003475, + 1.209875453998393, + 1.3080117649988097, + 1.4089794549981889, + 1.510417214998597, + 1.6066061399978935, + 1.7016627249977319, + 1.7983245360010187, + 1.8993615100007446, + 1.9964719649979088, + 2.088444521999918, + 2.164179654999316, + 2.223566459000722, + 2.3018017939975834, + 2.382025736998912, + 2.4783284939985606, + 2.5752275989980262, + 2.662720102998719, + 2.727756700998725, + 2.787365274998592, + 2.8645940339993103, + 2.9554433089979284, + 3.036471466999501, + 3.0967568140004005, + 3.165387824999925, + 3.257775660000334, + 3.362827932000073, + 3.4629307670002163, + 3.5696973019985307, + 3.6753393149992917, + 3.7769275379978353, + 3.878587028997572, + 3.974337530999037, + 4.069775894000486, + 4.168109374997584, + 4.265526839000813, + 4.367376315000001, + 4.463436122998246, + 4.55124707700088, + 4.612271071000578, + 4.687694886997633, + 4.7886533710006915, + 4.880237325000053, + 4.949422857000172, + 5.025821413000813, + 5.113766014001158, + 5.187554178999562, + 5.246507060001022, + 5.369932564997725 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.94404649881471, + "p95": 106.76653499831446, + "max": 462.35810699727153 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.7, + "per_stream_tps": 42.7, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.086, + "decode_tps": 42.688, + "prefill_tps": null, + "itl_mean_ms": 23.426, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.92, + "round_gap_p50_ms": 59.438, + "round_gap_p95_ms": 79.192, + "tokens_per_chunk": 2.56 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 23.426, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.92, + "round_gap_p50_ms": 59.438, + "round_gap_p95_ms": 79.192, + "tokens_per_chunk": 2.56 + }, + "timing_valid": true, + "end_to_end_tps": 42.2418197963834, + "makespan_s": 6.060344967001583, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.6880113205013, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.060344967001583, + "end_to_end_tps": 42.2418197963834, + "timing_valid": true, + "per_stream_tps": [ + 42.7 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08667190100095468, + "decode_tps": 42.6880113205013, + "prefill_tps": null, + "itl_mean_ms": 23.42578089412521, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.15996700001415, + "round_gap_p50_ms": 59.5083420012088, + "round_gap_p95_ms": 79.19233699794859, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.42578089412521, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.15996700001415, + "round_gap_p50_ms": 59.5083420012088, + "round_gap_p95_ms": 79.19233699794859, + "round_gap_mean_ms": 60.339132606080085, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29998.551553772, + "first": 29998.638225673, + "last": 30004.611799801, + "end": 30004.611898739, + "elapsed_s": 6.060344967001583, + "chunk_offsets_s": [ + 0.08667190100095468, + 0.14602722500058007, + 0.2148519300026237, + 0.29000374200040824, + 0.34763890400063246, + 0.39484909500242793, + 0.4427188160007063, + 0.4990421010006685, + 0.559595280999929, + 0.627271314002428, + 0.690532221000467, + 0.7601121180014161, + 0.8310834030016849, + 0.8906916340019961, + 0.9419683450032608, + 0.9976280040027632, + 1.0613213169999653, + 1.1229625760024646, + 1.1813385710011062, + 1.245594832002098, + 1.304137676001119, + 1.3656568530022923, + 1.4389031660030014, + 1.5240256050019525, + 1.5983344619999116, + 1.6561857710003096, + 1.707702830000926, + 1.7589187900011893, + 1.8053854740028328, + 1.8501692380013992, + 1.9100426490003883, + 1.974548127000162, + 2.0198168470014934, + 2.0664316560032603, + 2.1164792380004656, + 2.1689644640027836, + 2.215424107002036, + 2.2775691710012325, + 2.336907577002421, + 2.392702362001728, + 2.448777108002105, + 2.502165416000935, + 2.563626353003201, + 2.6248025109998707, + 2.6730143439999665, + 2.731249497002864, + 2.795007525000983, + 2.8553507110009377, + 2.9156634620012483, + 2.977308368001104, + 3.0373001780026243, + 3.0862383650019183, + 3.138300472000992, + 3.1842333109998435, + 3.2292762130018673, + 3.2757895900031144, + 3.3355604850003147, + 3.3945452270017995, + 3.454037776002224, + 3.5255262610007776, + 3.578214815002866, + 3.6390703080032836, + 3.7132302750032977, + 3.782114181001816, + 3.8577964830001292, + 3.9321179050020874, + 3.999671079000109, + 4.060192780001671, + 4.143292543001735, + 4.212840159001644, + 4.2603293230022246, + 4.305284699003096, + 4.3661412649998965, + 4.4408357860011165, + 4.508496231002937, + 4.567622777001816, + 4.627462974000082, + 4.67981262000103, + 4.734360203001415, + 4.790617516002385, + 4.8348480710010335, + 4.894356413002242, + 4.97451551800259, + 5.052547156003129, + 5.126179416001833, + 5.185609318999923, + 5.244647071001964, + 5.3238394079999125, + 5.402085510002507, + 5.484986942003161, + 5.555504154002847, + 5.609414865000872, + 5.665506588000426, + 5.723109560000012, + 5.781282651001675, + 5.836784572002216, + 5.884603463000531, + 5.932628854003269, + 5.993395460001921, + 6.060246029002883 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.5083420012088, + "p95": 79.19233699794859, + "max": 85.12243899895111 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.614090932938694, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.070218087003013, + "end_to_end_tps": 42.173114100813514, + "timing_valid": true, + "per_stream_tps": [ + 42.6 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08621512800164055, + "decode_tps": 42.614090932938694, + "prefill_tps": null, + "itl_mean_ms": 23.46641634509319, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.88438199996017, + "round_gap_p50_ms": 59.090706999995746, + "round_gap_p95_ms": 79.87105299980612, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.46641634509319, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.88438199996017, + "round_gap_p50_ms": 59.090706999995746, + "round_gap_p95_ms": 79.87105299980612, + "round_gap_mean_ms": 60.443799676755184, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30004.612569732, + "first": 30004.69878486, + "last": 30010.682721028, + "end": 30010.682787819, + "elapsed_s": 6.070218087003013, + "chunk_offsets_s": [ + 0.08621512800164055, + 0.145714427002531, + 0.21370907200252987, + 0.2910240689998318, + 0.3478370850025385, + 0.3959622570000647, + 0.44949931900191586, + 0.5044004360024701, + 0.5621590170012496, + 0.6282429590028187, + 0.693167972000083, + 0.7639389500000107, + 0.834233129000495, + 0.8931787710025674, + 0.9439806690024852, + 0.9999015980029071, + 1.0644791840022663, + 1.12612337900282, + 1.185964006002905, + 1.2491237260001071, + 1.3061174870017567, + 1.3702364380005747, + 1.4479712110005494, + 1.5313839260015811, + 1.6049073510002927, + 1.663696923002135, + 1.7164432050012692, + 1.767192311999679, + 1.813334399001178, + 1.8584578090012656, + 1.917731638000987, + 1.980483349001588, + 2.0258967080008006, + 2.0737598310006433, + 2.123744798002008, + 2.1741536950030422, + 2.2213577200018335, + 2.281838096001593, + 2.3403150850026577, + 2.3988427840013173, + 2.460180654001306, + 2.5125430890002463, + 2.5722599159998936, + 2.634281312002713, + 2.6830366290014354, + 2.741004415001953, + 2.8067998250007804, + 2.865352150001854, + 2.9277375180026866, + 2.991794272002153, + 3.0525178410025546, + 3.102106555001228, + 3.1522612780026975, + 3.1995269790022576, + 3.2450363080024545, + 3.2898264110008313, + 3.348917118000827, + 3.4099155250005424, + 3.4753434900012508, + 3.5424029040004825, + 3.591865966001933, + 3.6538950390022364, + 3.7287529010027356, + 3.7964758220005024, + 3.872413315002632, + 3.946095024002716, + 4.013572608000686, + 4.073478807000356, + 4.155127046000416, + 4.22420789300304, + 4.272246768003242, + 4.31757964800272, + 4.380134386003192, + 4.458364693000476, + 4.524688191002497, + 4.582374567002262, + 4.639455583001109, + 4.691498330001195, + 4.746759413003019, + 4.802737674002856, + 4.848009140001523, + 4.906773452003108, + 4.986644505002914, + 5.064740237001388, + 5.138624619001348, + 5.196676986000966, + 5.256532473002153, + 5.334367858002224, + 5.4161927530003595, + 5.502611348001665, + 5.570314303000487, + 5.621353772003204, + 5.6782571870026, + 5.733066030003101, + 5.791455042002781, + 5.847493134002434, + 5.89651248500013, + 5.94426435800051, + 6.00286766499994, + 6.070151296000404 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.090706999995746, + "p95": 79.87105299980612, + "max": 86.41859500130522 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.731295271319006, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.051704407000216, + "end_to_end_tps": 42.30213222309337, + "timing_valid": true, + "per_stream_tps": [ + 42.7 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08411230399724445, + "decode_tps": 42.731295271319006, + "prefill_tps": null, + "itl_mean_ms": 23.402052141190165, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.9202610020584, + "round_gap_p50_ms": 59.43833199853543, + "round_gap_p95_ms": 78.3526979976159, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.402052141190165, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.9202610020584, + "round_gap_p50_ms": 59.43833199853543, + "round_gap_p95_ms": 78.3526979976159, + "round_gap_mean_ms": 60.27801309094436, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30010.683411981, + "first": 30010.767524285, + "last": 30016.735047581, + "end": 30016.735116388, + "elapsed_s": 6.051704407000216, + "chunk_offsets_s": [ + 0.08411230399724445, + 0.14321312899846816, + 0.21279639799831784, + 0.28878510399954394, + 0.3460295639997639, + 0.3987681500002509, + 0.44908972600023844, + 0.501525928000774, + 0.558612548000383, + 0.6231357549986569, + 0.6887592879975273, + 0.7573566589999245, + 0.8284427759972459, + 0.8875223609975365, + 0.9390607549976266, + 0.9943065369989199, + 1.0587710500003595, + 1.1198995659979118, + 1.1805209209996974, + 1.2427997179984231, + 1.3017897919999086, + 1.3664475999976275, + 1.4440228549974563, + 1.5238330569991376, + 1.5974280249974981, + 1.6560857589975058, + 1.709610440000688, + 1.7600190230004955, + 1.8077654779990553, + 1.853259880997939, + 1.9114595969986112, + 1.9746494209975936, + 2.020041173000209, + 2.0682142339974234, + 2.1174892089984496, + 2.1676843499990355, + 2.213165213997854, + 2.272762202999729, + 2.3299804349990154, + 2.3930977300005907, + 2.4533055699976103, + 2.501889253999252, + 2.5602845110006456, + 2.621799453998392, + 2.6682852279991494, + 2.727723559997685, + 2.7933564109989675, + 2.8524057079994236, + 2.912496814999031, + 2.975451308000629, + 3.036340791000839, + 3.0853215269999055, + 3.135951718999422, + 3.1836808459993335, + 3.2284842720000597, + 3.274858368000423, + 3.3349806959995476, + 3.3991269109974382, + 3.4614976159973594, + 3.525896802999341, + 3.5763534649995563, + 3.6365456559979066, + 3.710465916999965, + 3.779450669997459, + 3.857008261998999, + 3.928774780000822, + 3.995130663999589, + 4.055411584999092, + 4.137944448997587, + 4.207218204999663, + 4.255539285997656, + 4.300515014998382, + 4.365228866998223, + 4.442341263998969, + 4.507085887998983, + 4.56283543599784, + 4.621094849997462, + 4.674362614998245, + 4.728580230999796, + 4.785430749998341, + 4.829738884000108, + 4.888987448997796, + 4.966313663000619, + 5.044666360998235, + 5.119326102998457, + 5.179135465998115, + 5.239931756997976, + 5.317133641998225, + 5.403390668998327, + 5.486220201000833, + 5.551195703999838, + 5.60293829299917, + 5.660849439998856, + 5.716900998999336, + 5.773547921999125, + 5.829462937999779, + 5.8792634019991965, + 5.926099784999678, + 5.9846140570007265, + 6.051635600000736 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.43833199853543, + "p95": 78.3526979976159, + "max": 86.25702700010152 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.3, + "per_stream_tps": 41.0, + "ttft_s_max": 0.17, + "request": { + "ttft_s": 0.127, + "decode_tps": 40.941, + "prefill_tps": null, + "itl_mean_ms": 24.427, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.186, + "round_gap_p50_ms": 58.714, + "round_gap_p95_ms": 80.396, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 24.425, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.998, + "round_gap_p50_ms": 58.784, + "round_gap_p95_ms": 80.316, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 80.55834177322099, + "makespan_s": 6.355642243001057, + "steady_tps": 82.2, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.33820459441586, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.3538444449986855, + "end_to_end_tps": 80.58113547350244, + "timing_valid": true, + "steady_tps": 82.20223665319479, + "per_stream_tps": [ + 41.2, + 40.7 + ], + "ttft_s": [ + 0.17, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.166506625002512, + "decode_tps": 41.21515292218173, + "prefill_tps": null, + "itl_mean_ms": 24.262921015678348, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.99510599882342, + "round_gap_p50_ms": 58.59605399746215, + "round_gap_p95_ms": 80.55321500069113, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08368669799892814, + "decode_tps": 40.67145269385473, + "prefill_tps": null, + "itl_mean_ms": 24.58727027842542, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.37617400157615, + "round_gap_p50_ms": 58.49705099899438, + "round_gap_p95_ms": 81.54434300013236, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.425095647051887, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.09732600010466, + "round_gap_p50_ms": 58.49705099899438, + "round_gap_p95_ms": 81.54434300013236, + "round_gap_mean_ms": 61.667320693051785, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30016.736521156, + "first": 30016.903027781, + "last": 30023.09007264, + "end": 30023.090114117, + "elapsed_s": 6.353592961000686, + "chunk_offsets_s": [ + 0.166506625002512, + 0.2280827640024654, + 0.2971007029991597, + 0.37932174500019755, + 0.44108513700120966, + 0.4915535180007282, + 0.5410479320016748, + 0.5955017310006951, + 0.6529885679992731, + 0.720493245000398, + 0.7850860670005204, + 0.8600811729993438, + 0.9329490109994367, + 0.994634106002195, + 1.0494309530004102, + 1.1095803000025626, + 1.1770070380007382, + 1.2439199069995084, + 1.3069050080011948, + 1.37717750000229, + 1.440418263002357, + 1.4998781060021429, + 1.5753056620014831, + 1.6574403820013686, + 1.734000485001161, + 1.797203915000864, + 1.8541651800005639, + 1.9084548850005376, + 1.9572630250004295, + 2.0053101490011613, + 2.059040634001576, + 2.122491516001901, + 2.1774781870008155, + 2.226272306001192, + 2.2805090340007155, + 2.3402178440010175, + 2.3921449389999907, + 2.4489976540025964, + 2.5034343250008533, + 2.55840408199947, + 2.61603364000257, + 2.6663641480008664, + 2.7193804920025286, + 2.7773644750013773, + 2.8270085500007553, + 2.881206280999322, + 2.935199780000403, + 2.9879261370006134, + 3.04609404599978, + 3.1113284300008672, + 3.1827832050003053, + 3.234393574999558, + 3.2882651729996724, + 3.3450251940012095, + 3.3946777480014134, + 3.442084114001773, + 3.4945505380019313, + 3.5482452670003113, + 3.601678573002573, + 3.6612730430024385, + 3.7125425890008046, + 3.771426925999549, + 3.8454452860023594, + 3.918682224000804, + 3.9992354390014953, + 4.077258244000404, + 4.150298671000201, + 4.20549380599914, + 4.279364285001066, + 4.360929079000925, + 4.423478247001185, + 4.469931645002362, + 4.528527698999824, + 4.596170519002044, + 4.662416323000798, + 4.717959019002592, + 4.778548801001307, + 4.839695897000638, + 4.896330804000172, + 4.951362239000446, + 5.0113873289992625, + 5.058908817001793, + 5.111702045000129, + 5.1829682490024425, + 5.2633199369993235, + 5.348714140000084, + 5.416082047002419, + 5.471563810002408, + 5.540840188001312, + 5.6186778280025464, + 5.703156960000342, + 5.772178619001352, + 5.828084177999699, + 5.88970452400099, + 5.948605213001429, + 6.004390165002405, + 6.0611508790025255, + 6.1141991290023725, + 6.165108893001161, + 6.218498225000076, + 6.273557713000628, + 6.353551484000491 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.59605399746215, + "p95": 80.55321500069113, + "max": 85.39420300076017 + } + }, + { + "start": 30016.736269672, + "first": 30016.81995637, + "last": 30023.089710291, + "end": 30023.089776668, + "elapsed_s": 6.353506995998032, + "chunk_offsets_s": [ + 0.08368669799892814, + 0.22861964099865872, + 0.2977646500003175, + 0.3799464879994048, + 0.44142210299833096, + 0.49159246599811013, + 0.5410894550004741, + 0.5959968429997389, + 0.6524475289988914, + 0.720416338997893, + 0.7849329599994235, + 0.8600302859995281, + 0.9328306280003744, + 0.9949534699990181, + 1.049469677000161, + 1.1093632339980104, + 1.1770937029978086, + 1.243941796998115, + 1.307299484997202, + 1.377684101997147, + 1.440494206999574, + 1.5003804189982475, + 1.5757565929998236, + 1.6579448390002653, + 1.7344187719972979, + 1.7972967089990561, + 1.8546165109983122, + 1.9090146609996737, + 1.9573256709991256, + 2.0057395580006414, + 2.058935275999829, + 2.1228532530003577, + 2.1776507979993767, + 2.226480952998827, + 2.280462497998087, + 2.3401822060004633, + 2.3921481859979394, + 2.4495958189982048, + 2.50343672300005, + 2.5584373639976548, + 2.616091181000229, + 2.6668065499980003, + 2.719960557999002, + 2.7775546880002366, + 2.827302214998781, + 2.881359512997733, + 2.9354148589991382, + 2.9883556259992474, + 3.046285298998555, + 3.1116197109986388, + 3.1829687770004966, + 3.2345603609974205, + 3.2887568939986522, + 3.3451982049991784, + 3.3952159699983895, + 3.4424756789994717, + 3.4950905039986537, + 3.548765758998343, + 3.601845566998236, + 3.661485499000264, + 3.7127526119984395, + 3.7715318970003864, + 3.8456105189980008, + 3.9192068289994495, + 3.9997183269988454, + 4.077748604999215, + 4.150511094998365, + 4.205944514000294, + 4.279699266997341, + 4.361243609997473, + 4.423800251999637, + 4.47022742199988, + 4.528724472998874, + 4.596485050999036, + 4.662611384999764, + 4.718153664998681, + 4.7788584679983614, + 4.839998106999701, + 4.896493093998288, + 4.951497116999235, + 5.011585366999498, + 5.059074786000565, + 5.111888674000511, + 5.183076787998289, + 5.263444606000121, + 5.348846090000734, + 5.416060393999942, + 5.471943934000592, + 5.5411454389977735, + 5.619238453000435, + 5.704266532000474, + 5.77268645499862, + 5.828541495000536, + 5.890256908998708, + 5.948789424997813, + 6.004884221998509, + 6.061252488998434, + 6.114206342997932, + 6.165131132998795, + 6.21844032700028, + 6.2740572599977895, + 6.353440618997411 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.49705099899438, + "p95": 81.54434300013236, + "max": 144.93294299973059 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.35599662627337, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.355642243001057, + "end_to_end_tps": 80.55834177322099, + "timing_valid": true, + "steady_tps": 82.2114380296359, + "per_stream_tps": [ + 41.2, + 40.7 + ], + "ttft_s": [ + 0.17, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16897372799940058, + "decode_tps": 41.22218058340695, + "prefill_tps": null, + "itl_mean_ms": 24.258784611761346, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.55184700302198, + "round_gap_p50_ms": 59.743433997937245, + "round_gap_p95_ms": 78.61799400052405, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08680812400052673, + "decode_tps": 40.67799831313668, + "prefill_tps": null, + "itl_mean_ms": 24.58331386667708, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.56475299861631, + "round_gap_p50_ms": 59.52671799968812, + "round_gap_p95_ms": 80.7003240006452, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.421049239219215, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.99787000051583, + "round_gap_p50_ms": 59.52671799968812, + "round_gap_p95_ms": 80.31627499804017, + "round_gap_mean_ms": 61.65710451486039, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30023.091055096, + "first": 30023.260028824, + "last": 30029.4460189, + "end": 30029.446135315, + "elapsed_s": 6.355080219000229, + "chunk_offsets_s": [ + 0.16897372799940058, + 0.2319837409995671, + 0.3012235400019563, + 0.379802194001968, + 0.43989426500047557, + 0.49058966799930204, + 0.5400904919988534, + 0.5963711529984721, + 0.6542895509992377, + 0.7230668769989279, + 0.7878688310011057, + 0.8619790869997814, + 0.9378294950001873, + 1.0073669779994816, + 1.0640568589988106, + 1.1247323679999681, + 1.192068812000798, + 1.2580138519988395, + 1.31634870400012, + 1.3833035329989798, + 1.445009648999985, + 1.5052959579988965, + 1.5798478050019185, + 1.6615474300015194, + 1.7370625319999817, + 1.8013416950016108, + 1.8567034430016065, + 1.9119999340000504, + 1.965523437000229, + 2.015693370001827, + 2.0700524800013227, + 2.1320990770000208, + 2.1849822280019, + 2.2350570609996794, + 2.287293362998753, + 2.3426688090003154, + 2.392516477000754, + 2.449305458998424, + 2.503273545000411, + 2.5582271780003794, + 2.6159971300003235, + 2.6663425719998486, + 2.7190135950004333, + 2.776886840998486, + 2.8268487070017727, + 2.8813884619994496, + 2.936842394999985, + 2.9968404840001313, + 3.059718182001234, + 3.1270413199999894, + 3.198611936000816, + 3.250163489999977, + 3.3028274409989535, + 3.353447726000013, + 3.400647309001215, + 3.448001487999136, + 3.5010162830003537, + 3.554581849999522, + 3.6079799569997704, + 3.6679009799991036, + 3.7198533000009775, + 3.777937499999098, + 3.8530767439988267, + 3.926939041000878, + 4.013186141000915, + 4.094669997000892, + 4.166613346998929, + 4.222422986000311, + 4.295309704000829, + 4.371567889000289, + 4.433040919000632, + 4.47969010000088, + 4.536387259999174, + 4.603855929999554, + 4.670376048001344, + 4.72626094700172, + 4.786040963001142, + 4.848991959999694, + 4.903592769001989, + 4.965665365998575, + 5.02786196300076, + 5.077241128001333, + 5.131164122998598, + 5.1994690370011085, + 5.279785311999149, + 5.358271049000905, + 5.421611472000222, + 5.477319494999392, + 5.545902075999038, + 5.624520069999562, + 5.709290337999846, + 5.77813046600204, + 5.834725239001273, + 5.8962727620018995, + 5.9565490819986735, + 6.0147284750019026, + 6.07447190899984, + 6.126815669998905, + 6.176298484999279, + 6.230735891000222, + 6.286137565999525, + 6.354963803998544 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.743433997937245, + "p95": 78.61799400052405, + "max": 86.24710000003688 + } + }, + { + "start": 30023.090965645, + "first": 30023.177773769, + "last": 30029.446518805, + "end": 30029.446607888, + "elapsed_s": 6.355642243001057, + "chunk_offsets_s": [ + 0.08680812400052673, + 0.23173908700118773, + 0.3009140620015387, + 0.3795231040021463, + 0.43991857200308004, + 0.49028386999998474, + 0.5398878270025307, + 0.5959769130022323, + 0.6539104010007577, + 0.7230296400011866, + 0.787679000000935, + 0.8618494069996814, + 0.9377077680001094, + 1.007276055002876, + 1.0642534110011184, + 1.125153892000526, + 1.192104511999787, + 1.2581969940001727, + 1.3160314010019647, + 1.3838139480030804, + 1.4449690680012282, + 1.5051323700026842, + 1.5801302400032, + 1.6614716290023352, + 1.7372141670020937, + 1.801369426000747, + 1.8567113790013536, + 1.912158178001846, + 1.9657334310031729, + 2.015942393001751, + 2.0700909970000794, + 2.1322376700009045, + 2.1850114799999574, + 2.2354505180010165, + 2.287625611999829, + 2.3428186350029137, + 2.392824403003033, + 2.449100633002672, + 2.5031657710023865, + 2.558377820001624, + 2.6160160600011295, + 2.666303415000584, + 2.718608712002606, + 2.7768933050028863, + 2.826896153001144, + 2.8814123200027097, + 2.937118317000568, + 2.9971152700018138, + 3.060051648000808, + 3.1270669069999713, + 3.198538919001294, + 3.250157377002324, + 3.3028366580001602, + 3.353500660999998, + 3.4009633650020987, + 3.448312311000336, + 3.501311328000156, + 3.5546338570020453, + 3.608303518001776, + 3.667933720000292, + 3.7198471070005326, + 3.7779683680018934, + 3.8535331210005097, + 3.927189312002156, + 4.013219473003119, + 4.094481797001208, + 4.166929995000828, + 4.2222256010027195, + 4.295312440000998, + 4.3719559220007795, + 4.433322250002675, + 4.479735594002705, + 4.536323252003058, + 4.603893071001949, + 4.670417925000947, + 4.726403173001017, + 4.785999597002956, + 4.848788750001404, + 4.903622453002754, + 4.965829371001746, + 5.027900111002964, + 5.077284717001021, + 5.131583584003238, + 5.199457835002249, + 5.2801581590028945, + 5.358478755002579, + 5.42163697900105, + 5.477160482001636, + 5.54589916400073, + 5.624298632999853, + 5.709145567001542, + 5.778286838001804, + 5.834699779003131, + 5.896174253000936, + 5.956792857003165, + 6.014902113001881, + 6.074428831001569, + 6.126767503003066, + 6.176340281999728, + 6.23075691700069, + 6.2861786740031675, + 6.355553160003183 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.52671799968812, + "p95": 80.7003240006452, + "max": 144.930963000661 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.33352442335145, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.355665931998374, + "end_to_end_tps": 80.55804151415097, + "timing_valid": true, + "steady_tps": 82.17361872326708, + "per_stream_tps": [ + 40.7, + 41.2 + ], + "ttft_s": [ + 0.08, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08491037700150628, + "decode_tps": 40.666762211675724, + "prefill_tps": null, + "itl_mean_ms": 24.590106160772564, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.16439399862429, + "round_gap_p50_ms": 58.832880000409205, + "round_gap_p95_ms": 80.23829499870772, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1662264829974447, + "decode_tps": 41.203217355004206, + "prefill_tps": null, + "itl_mean_ms": 24.269949392157073, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.12238899976364, + "round_gap_p50_ms": 58.512597999651916, + "round_gap_p95_ms": 78.83351399868843, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.43002777646482, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.12238899976364, + "round_gap_p50_ms": 58.78439300067839, + "round_gap_p95_ms": 80.23829499870772, + "round_gap_mean_ms": 61.679773098995334, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30029.448290761, + "first": 30029.533201138, + "last": 30035.803678209, + "end": 30035.80376006, + "elapsed_s": 6.355469299000106, + "chunk_offsets_s": [ + 0.08491037700150628, + 0.22905788400021265, + 0.29874274499889, + 0.3760994030017173, + 0.43708226200033096, + 0.48761040099998354, + 0.538823973001854, + 0.5988624839992553, + 0.6591317620004702, + 0.7294764940015739, + 0.7950185699992289, + 0.8676729780017922, + 0.9399282949998451, + 1.0033618560009927, + 1.0595946089997597, + 1.1185520520011778, + 1.1847945070003334, + 1.2505066039993835, + 1.3106532099991455, + 1.3769302209984744, + 1.4393284479992872, + 1.4981128409999656, + 1.5743741650003358, + 1.6637250290004886, + 1.741898124000727, + 1.803168940001342, + 1.8588133529992774, + 1.9138377339986619, + 1.9640630370013241, + 2.0112062570005946, + 2.065264174998447, + 2.129365777000203, + 2.1834460019999824, + 2.233649814999808, + 2.2851792729998124, + 2.3400998809993325, + 2.3891136839993123, + 2.446291429001576, + 2.500340664999385, + 2.557523465999111, + 2.622729808001168, + 2.674839603001601, + 2.7281908270015265, + 2.7858809270001075, + 2.8344520309983636, + 2.887470107001718, + 2.9410256929986645, + 2.9952741070010234, + 3.0556236150005134, + 3.122354382001504, + 3.1939814280012797, + 3.2441779940018023, + 3.2986893390007026, + 3.3486502220002876, + 3.3969542830018327, + 3.444389683998452, + 3.4971022680001624, + 3.5523804730000847, + 3.610371456001303, + 3.673855952998565, + 3.725453365001158, + 3.7834226160011895, + 3.8595870099998137, + 3.9320533589998377, + 4.012291653998545, + 4.090428198000154, + 4.162348617999669, + 4.217860470998858, + 4.2895270330009225, + 4.366005799998675, + 4.427840202999505, + 4.475757140000496, + 4.533329338999465, + 4.605620744001499, + 4.676382295998337, + 4.735215175998746, + 4.793692601000657, + 4.854777016000298, + 4.908852521999506, + 4.964393376998487, + 5.023475458998291, + 5.071384619001037, + 5.124834233000001, + 5.1947359740006505, + 5.275815901000897, + 5.354773672999727, + 5.419869440000184, + 5.475220946998888, + 5.543508154001756, + 5.627495976001228, + 5.713578178001626, + 5.782736638000642, + 5.839549174001149, + 5.900507791000564, + 5.9599739980003505, + 6.014052517999517, + 6.071933438000997, + 6.125445369001682, + 6.175014336000459, + 6.22962812699916, + 6.284164676999353, + 6.35538744799851 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.832880000409205, + "p95": 80.23829499870772, + "max": 144.14750699870638 + } + }, + { + "start": 30029.448094128, + "first": 30029.614320611, + "last": 30035.803157706, + "end": 30035.803287068, + "elapsed_s": 6.355192939998233, + "chunk_offsets_s": [ + 0.1662264829974447, + 0.22943500399924233, + 0.2986542849976104, + 0.3764189319990692, + 0.4371666399965761, + 0.48805335399811156, + 0.5392873439996038, + 0.5992726329968718, + 0.6595762029974139, + 0.729898116998811, + 0.7949405269973795, + 0.8680629159971431, + 0.9402907249968848, + 1.003836700998363, + 1.0601554649983882, + 1.1185479549967567, + 1.185371668998414, + 1.250926945998799, + 1.3106088679996901, + 1.3768407519964967, + 1.4393560190001153, + 1.4985232619983435, + 1.574518535999232, + 1.6637985259985726, + 1.74181270400004, + 1.8036646199980169, + 1.8588887070000055, + 1.913790863000031, + 1.9639044559989998, + 2.0112281469992013, + 2.0651802590000443, + 2.129160789998423, + 2.183383864998177, + 2.2336674639991543, + 2.285495345997333, + 2.340449446997809, + 2.3891138109975145, + 2.446327688998281, + 2.5003307579972898, + 2.5578034379977908, + 2.6228074809987447, + 2.6752850849989045, + 2.7286350759968627, + 2.7858252070000162, + 2.8349147310000262, + 2.887370556996757, + 2.941015050997521, + 2.9952696899999864, + 3.0559656109981006, + 3.122445112996502, + 3.19411131599918, + 3.244481008998264, + 3.2989872170001036, + 3.3487877429979562, + 3.397619201998168, + 3.4444747339985042, + 3.497594267999375, + 3.552655628998764, + 3.6107048589983606, + 3.6739834410000185, + 3.7258908779986086, + 3.7834985609988507, + 3.8600152259969036, + 3.9320841139997356, + 4.012404931996571, + 4.091053111998917, + 4.162436468999658, + 4.217733844998293, + 4.2896070269998745, + 4.366488263000065, + 4.428198041998257, + 4.4761107379999885, + 4.5339372269991145, + 4.6059263039969665, + 4.6762963789988135, + 4.735623485998076, + 4.794040437998774, + 4.85521523299758, + 4.909305108998524, + 4.964995231999637, + 5.023507829999289, + 5.071272761997534, + 5.12484172099721, + 5.195112678997248, + 5.27587288399809, + 5.354706397996779, + 5.420228142997075, + 5.47531250999964, + 5.543321230998117, + 5.627385575997323, + 5.71387668099851, + 5.782627405998937, + 5.839533522997954, + 5.900946824996936, + 5.959998223999719, + 6.014016047000041, + 6.071930763999262, + 6.1254553369981295, + 6.1750203829979, + 6.2296035299987125, + 6.284787126998708, + 6.355063577997498 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.512597999651916, + "p95": 78.83351399868843, + "max": 89.27998999934061 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 140.8, + "per_stream_tps": 35.9, + "ttft_s_max": 0.35, + "request": { + "ttft_s": 0.212, + "decode_tps": 35.837, + "prefill_tps": null, + "itl_mean_ms": 27.905, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.593, + "round_gap_p50_ms": 65.765, + "round_gap_p95_ms": 91.948, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 27.915, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.471, + "round_gap_p50_ms": 65.764, + "round_gap_p95_ms": 92.278, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 139.69972980189297, + "makespan_s": 7.330007018998003, + "steady_tps": 145.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 140.62835876231628, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.339804059000016, + "end_to_end_tps": 139.51326108554332, + "timing_valid": true, + "steady_tps": 145.00841459922438, + "per_stream_tps": [ + 36.5, + 35.2, + 35.6, + 36.0 + ], + "ttft_s": [ + 0.35, + 0.09, + 0.17, + 0.26 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.34738399799971376, + "decode_tps": 36.47678020431162, + "prefill_tps": null, + "itl_mean_ms": 27.41470037648219, + "itl_p50_ms": 0.0, + "itl_p95_ms": 85.93313399978797, + "round_gap_p50_ms": 65.82606799929636, + "round_gap_p95_ms": 91.72105200195801, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08579374900000403, + "decode_tps": 35.15722566310354, + "prefill_tps": null, + "itl_mean_ms": 28.443655070583976, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.10849600154324, + "round_gap_p50_ms": 65.9528539981693, + "round_gap_p95_ms": 93.08775400131708, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16807821999827866, + "decode_tps": 35.564697835677855, + "prefill_tps": null, + "itl_mean_ms": 28.117770172556288, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.45332299784059, + "round_gap_p50_ms": 65.51181599934353, + "round_gap_p95_ms": 92.72755500205676, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25528467199910665, + "decode_tps": 35.997050364424574, + "prefill_tps": null, + "itl_mean_ms": 27.780053917647855, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.30386300137616, + "round_gap_p50_ms": 65.76433499867562, + "round_gap_p95_ms": 92.24116899713408, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.93904488431758, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.10849600154324, + "round_gap_p50_ms": 65.76433499867562, + "round_gap_p95_ms": 92.44222399865976, + "round_gap_mean_ms": 70.53917272773249, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30035.805718727, + "first": 30036.153102725, + "last": 30043.143851321, + "end": 30043.144789422, + "elapsed_s": 7.339070695001283, + "chunk_offsets_s": [ + 0.34738399799971376, + 0.4148667120025493, + 0.4914289810003538, + 0.5807847180003591, + 0.6483753829998022, + 0.7040594200007035, + 0.7590219680023438, + 0.8210974499997974, + 0.8850733670005866, + 0.9630763599998318, + 1.034496524000133, + 1.1227914959999907, + 1.2039914640008647, + 1.2843639900020207, + 1.3499302650016034, + 1.4152585240008193, + 1.491596258001664, + 1.5682575240025471, + 1.6312308680026035, + 1.7098249120026594, + 1.7813850790007564, + 1.8461364110007707, + 1.9323435930018604, + 2.024690439000551, + 2.110341720002907, + 2.1833228540017444, + 2.253250255002058, + 2.3155245689995354, + 2.3708124799995858, + 2.4232725760011817, + 2.4814786650022143, + 2.548833888002264, + 2.607735175999551, + 2.661713162000524, + 2.720106573000521, + 2.7823094529994705, + 2.8386079670017352, + 2.900715026000398, + 2.9643635770007677, + 3.0263633340000524, + 3.0939720290007244, + 3.1507181609995314, + 3.208171782000136, + 3.2795257970028615, + 3.336017781002738, + 3.3935740090018953, + 3.452854811999714, + 3.5116131260001566, + 3.576118895001855, + 3.6524945180026407, + 3.7359309460007353, + 3.792526576002274, + 3.850146685002983, + 3.9098871520000102, + 3.9619631950008625, + 4.015418997001689, + 4.075674517000152, + 4.136395527002605, + 4.194889296999463, + 4.271628512000461, + 4.3327871180008515, + 4.398044937002851, + 4.484587986000406, + 4.570002124000894, + 4.662444347999553, + 4.753571637000277, + 4.8366881260008086, + 4.901061118001962, + 4.981315160999657, + 5.067248294999445, + 5.136913784001081, + 5.189256069003022, + 5.259198316001857, + 5.335537574002956, + 5.411621405000915, + 5.4745342480018735, + 5.54036031600117, + 5.611956636003015, + 5.673975961002725, + 5.740083814001991, + 5.8080689380003605, + 5.8604851529999, + 5.920416937002301, + 5.999682495999878, + 6.093187192000187, + 6.184908244002145, + 6.267546172002767, + 6.334518122999725, + 6.410606565001217, + 6.499147407001146, + 6.595690197002114, + 6.67247972900077, + 6.736431717999949, + 6.806829610002751, + 6.870800289001636, + 6.929903512002056, + 6.996906763000879, + 7.056731291002507, + 7.110956798002007, + 7.169315118000668, + 7.23313426200184, + 7.338132594002673 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.82606799929636, + "p95": 91.72105200195801, + "max": 104.9983320008323 + } + }, + { + "start": 30035.80522994, + "first": 30035.891023689, + "last": 30043.144155732, + "end": 30043.144445869, + "elapsed_s": 7.339215928997874, + "chunk_offsets_s": [ + 0.08579374900000403, + 0.4149721510002564, + 0.49254686199856224, + 0.5809287379997841, + 0.6486508130001312, + 0.7041780129984545, + 0.7591134209978918, + 0.8211939760003588, + 0.8850929709988122, + 0.9643402929978038, + 1.0346501290005108, + 1.1225554949996877, + 1.2048706719979236, + 1.2850044619990513, + 1.3503171179982019, + 1.4158451319999585, + 1.4910156340010872, + 1.5679630519989587, + 1.6311894229984318, + 1.709701806998055, + 1.7813789030005864, + 1.8458807919996616, + 1.9319892880012048, + 2.0254986779982573, + 2.1104118090006523, + 2.1841012650002085, + 2.253509778998705, + 2.3166161429981003, + 2.371794262999174, + 2.4241131560011127, + 2.481705784000951, + 2.5488073240012454, + 2.6076792159983597, + 2.6616885990006267, + 2.7199606649992347, + 2.7822695590002695, + 2.838429269999324, + 2.9006607459996303, + 2.9644607580012234, + 3.027427640998212, + 3.0937342350007384, + 3.1506483259981906, + 3.2090022490010597, + 3.2798907430005784, + 3.3365992689978157, + 3.3934849069992197, + 3.4538328829985403, + 3.512041121000948, + 3.5770519510006125, + 3.6527638509978715, + 3.736381151000387, + 3.792901314998744, + 3.8504954839991115, + 3.9102085230006196, + 3.9624093679994985, + 4.015816027000255, + 4.076013410998712, + 4.13725495700055, + 4.1955536770001345, + 4.272381047998351, + 4.333370078998996, + 4.3984083629984525, + 4.484676636999211, + 4.5697641859987925, + 4.66285194000011, + 4.754253715000232, + 4.837049408000894, + 4.901612122001097, + 4.981753539999772, + 5.067819285999576, + 5.13733665700056, + 5.190164866999112, + 5.259767338000529, + 5.335995524997998, + 5.412147284998355, + 5.4748589800001355, + 5.540811833998305, + 5.612350177998451, + 5.674129660998005, + 5.740121416998591, + 5.808437052000954, + 5.86074467599974, + 5.920955587000208, + 6.0001091140002245, + 6.093579534001037, + 6.185877466999955, + 6.26799341700098, + 6.335396213999047, + 6.411169473998598, + 6.500425424997957, + 6.596249057998648, + 6.672930542998074, + 6.736687175998668, + 6.807012049997866, + 6.8711521129989706, + 6.9304350889979105, + 6.996997588998056, + 7.056752378997771, + 7.112018466999871, + 7.16975285800072, + 7.233158022998396, + 7.338925791998918 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.9528539981693, + "p95": 93.08775400131708, + "max": 329.1784020002524 + } + }, + { + "start": 30035.805466196, + "first": 30035.973544416, + "last": 30043.14357581, + "end": 30043.143654461, + "elapsed_s": 7.338188264999189, + "chunk_offsets_s": [ + 0.16807821999827866, + 0.4150696529977722, + 0.4922228180003003, + 0.5812140089983586, + 0.6487561229987477, + 0.7044893829988723, + 0.7593244889976631, + 0.8215324379998492, + 0.8857157269994786, + 0.9635169009998208, + 1.0347967729976517, + 1.1228518879979674, + 1.2044526629979373, + 1.2849735299969325, + 1.349857951998274, + 1.4151339629970607, + 1.491165766998165, + 1.5680712749999657, + 1.63117542100008, + 1.7097855789979803, + 1.7815105369991215, + 1.8459843579985318, + 1.932047487996897, + 2.0247750429989537, + 2.1104869239970867, + 2.184042810000392, + 2.2538675399991916, + 2.315871672999492, + 2.3717590579981334, + 2.423808105999342, + 2.481945817999076, + 2.548725952998211, + 2.6077411139995093, + 2.661763426000107, + 2.719967433997226, + 2.782255157999316, + 2.838662950998696, + 2.900630246000219, + 2.964430594998703, + 3.0272861019984703, + 3.0936735559989756, + 3.1510301239977707, + 3.2088391069992213, + 3.279493199999706, + 3.336086957999214, + 3.393562758999906, + 3.4534176259985543, + 3.512028064000333, + 3.576306281000143, + 3.653063972997188, + 3.736357452999073, + 3.79291942999771, + 3.85036629899696, + 3.91042311399724, + 3.9623742759977176, + 4.015385279999464, + 4.0761223239969695, + 4.137193957998534, + 4.195083388000057, + 4.272195630997885, + 4.333001179998973, + 4.398441584999091, + 4.484894907996932, + 4.569637272998079, + 4.6628787129993725, + 4.754357041998446, + 4.836753878997115, + 4.901778961997479, + 4.9811535849985376, + 5.068136819998472, + 5.137320464000368, + 5.189659149000363, + 5.259706754997751, + 5.335674456997367, + 5.411674131999462, + 5.474870630998339, + 5.540382446997683, + 5.612066474997846, + 5.673997308000253, + 5.739772777000326, + 5.808353952998004, + 5.860581341999932, + 5.9208748409982945, + 6.000032337000448, + 6.093398916997103, + 6.185032916997443, + 6.267587776997971, + 6.334701268999197, + 6.41090508599882, + 6.499748754999018, + 6.595553936997021, + 6.672854404998361, + 6.736870786997315, + 6.806524381998315, + 6.871221771998535, + 6.930529903998831, + 6.997000502997253, + 7.056656494998606, + 7.111611754997284, + 7.169728759999998, + 7.23328991499875, + 7.338109614000132 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.51181599934353, + "p95": 92.72755500205676, + "max": 246.99143299949355 + } + }, + { + "start": 30035.804985363, + "first": 30036.060270035, + "last": 30043.144183784, + "end": 30043.144296472, + "elapsed_s": 7.339311109000846, + "chunk_offsets_s": [ + 0.25528467199910665, + 0.4156297759982408, + 0.492216193000786, + 0.5812395319990173, + 0.648983513998246, + 0.7047488739990513, + 0.7596418760003871, + 0.8216328989983595, + 0.8855770039990603, + 0.9643666160009161, + 1.0349430329988536, + 1.1230824790000042, + 1.204810567000095, + 1.2851891110003635, + 1.3501601999996637, + 1.4159245349983394, + 1.4921288900004583, + 1.5688062389999686, + 1.6321223500017368, + 1.7108662220016413, + 1.78237153399823, + 1.8471687919991382, + 1.9333602430015162, + 2.0256014119986503, + 2.1108883540000534, + 2.1841548880001938, + 2.254450420001376, + 2.317053130998829, + 2.371624879000592, + 2.423882146998949, + 2.4823044260010647, + 2.5493445329993847, + 2.6080482750003284, + 2.662398807999125, + 2.7206192830017244, + 2.7828106410015607, + 2.8389420439998503, + 2.9012921120011015, + 2.9650648569986515, + 3.027418918998592, + 3.0943770759986364, + 3.1511192700018, + 3.209559717000957, + 3.2800626380012545, + 3.336648314998456, + 3.3943706930003827, + 3.4537167540001974, + 3.512697786998615, + 3.577448389001802, + 3.652930609001487, + 3.7365361160009343, + 3.7932260790003056, + 3.8509431290003704, + 3.9106708750005055, + 3.962561595999432, + 4.015904597999906, + 4.076312844001222, + 4.137300498998229, + 4.195902283998294, + 4.272453425001004, + 4.333562248000817, + 4.398850438999943, + 4.48515430200132, + 4.570848816001671, + 4.663595738998993, + 4.754427402000147, + 4.837475497999549, + 4.902554571999644, + 4.981939853001677, + 5.068259153998952, + 5.137735352000163, + 5.1905667780010845, + 5.260278928999469, + 5.336098994001077, + 5.412190602000919, + 5.475417232999462, + 5.540812760998961, + 5.612658043999545, + 5.674634259001323, + 5.741000131001783, + 5.808734387999721, + 5.8612493539985735, + 5.921321845999046, + 6.000551671000721, + 6.093731329001457, + 6.185922976001166, + 6.2681791210015945, + 6.335920189998433, + 6.411925226999301, + 6.500484280000819, + 6.596371456998895, + 6.673129881000932, + 6.736888834999263, + 6.807214956999815, + 6.871894455998699, + 6.930915619999723, + 6.997055083000305, + 7.0571952239988605, + 7.1116057180006464, + 7.170657080001547, + 7.233594754998194, + 7.33919842099931 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.76433499867562, + "p95": 92.24116899713408, + "max": 160.34510399913415 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 140.81638691009442, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.330007018998003, + "end_to_end_tps": 139.69972980189297, + "timing_valid": true, + "steady_tps": 144.9701004956355, + "per_stream_tps": [ + 35.6, + 36.0, + 35.2, + 36.5 + ], + "ttft_s": [ + 0.17, + 0.25, + 0.09, + 0.34 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16815807000239147, + "decode_tps": 35.61168650029984, + "prefill_tps": null, + "itl_mean_ms": 28.080669529413615, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.44374799789512, + "round_gap_p50_ms": 65.85127000289503, + "round_gap_p95_ms": 91.44132700021146, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25184626600093907, + "decode_tps": 36.037635101218996, + "prefill_tps": null, + "itl_mean_ms": 27.748768674506458, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.17250900008366, + "round_gap_p50_ms": 65.87352999849827, + "round_gap_p95_ms": 91.69462800127803, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08620786599931307, + "decode_tps": 35.207595942450446, + "prefill_tps": null, + "itl_mean_ms": 28.40296172549179, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.32918799796607, + "round_gap_p50_ms": 65.76480500007165, + "round_gap_p95_ms": 91.63884300141945, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33601107299909927, + "decode_tps": 36.46099849679474, + "prefill_tps": null, + "itl_mean_ms": 27.426566501954394, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.61331699840957, + "round_gap_p50_ms": 65.8675550002954, + "round_gap_p95_ms": 90.99764599886839, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.914741607841563, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.623346000124, + "round_gap_p50_ms": 65.83867600056692, + "round_gap_p95_ms": 91.63884300141945, + "round_gap_mean_ms": 70.47781297029306, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30043.146127357, + "first": 30043.314285427, + "last": 30050.474856157, + "end": 30050.475273242, + "elapsed_s": 7.329145884999889, + "chunk_offsets_s": [ + 0.16815807000239147, + 0.4037856280010601, + 0.4810523500018462, + 0.5705310170014855, + 0.6399130010031513, + 0.6940990280018013, + 0.7486899610012188, + 0.8124717780010542, + 0.8770232090028003, + 0.9647537020027812, + 1.0376498630030255, + 1.1250936110009206, + 1.206941293003183, + 1.2816797660016164, + 1.3455299449997256, + 1.4118563340016408, + 1.4882164150003518, + 1.5637096040009055, + 1.6279539830029535, + 1.7047102700016694, + 1.774705450003239, + 1.8377483170006599, + 1.9275840490008704, + 2.023771703999955, + 2.1071011400017596, + 2.1793816730023536, + 2.2440034710016334, + 2.3035981620014354, + 2.3601353730009578, + 2.4136307380031212, + 2.471263899002224, + 2.540997760999744, + 2.6012150380011008, + 2.6551427480007987, + 2.7132832930001314, + 2.7761709580008755, + 2.8345942000014475, + 2.9034523280024587, + 2.9718772580017685, + 3.037576385002467, + 3.1028610690009373, + 3.1583601430029375, + 3.2167322740024247, + 3.2843513110019558, + 3.3381625410002016, + 3.39739255600216, + 3.45682504800061, + 3.5153020580000884, + 3.5802498600023682, + 3.654860994000046, + 3.7402773320027336, + 3.795286844000657, + 3.8556385100018815, + 3.918816459001391, + 3.973018413002137, + 4.0269682180005475, + 4.085990442003094, + 4.146517750003113, + 4.204696091001097, + 4.273078221001924, + 4.330024095001136, + 4.394773681000515, + 4.480956371000502, + 4.567119041999831, + 4.661019096001837, + 4.751069011999789, + 4.835917878001055, + 4.905021545000636, + 4.990649070001382, + 5.07717994199993, + 5.143586734000564, + 5.194959357002517, + 5.259616244002245, + 5.332070625001506, + 5.40701516199988, + 5.471189435000269, + 5.5370407050031645, + 5.606637964003312, + 5.668077202000859, + 5.7331773250007245, + 5.8025400590013305, + 5.855322225001146, + 5.91922791700199, + 6.002267079002195, + 6.093708406002406, + 6.184624967001582, + 6.262144806001743, + 6.325289345000783, + 6.399527212000976, + 6.488926159003313, + 6.586207545002253, + 6.664750001000357, + 6.728677904000506, + 6.800423172000592, + 6.8658889180005644, + 6.932834148003167, + 7.001409043001331, + 7.06125503000294, + 7.116998266999872, + 7.17445298900202, + 7.234917420002603, + 7.328728800002864 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.85127000289503, + "p95": 91.44132700021146, + "max": 235.62755799866864 + } + }, + { + "start": 30043.14644529, + "first": 30043.398291556, + "last": 30050.474227568, + "end": 30050.474301755, + "elapsed_s": 7.3278564649990585, + "chunk_offsets_s": [ + 0.25184626600093907, + 0.4035186619985325, + 0.48078635999991093, + 0.570478641999216, + 0.6393660440007807, + 0.6938537770001858, + 0.7484402139998565, + 0.8119947179984592, + 0.8769655850010167, + 0.9641380940011004, + 1.037026125999546, + 1.1243662429988035, + 1.2065887470016605, + 1.2814680400006182, + 1.3454672480002046, + 1.411596648998966, + 1.4875931259994104, + 1.5632502590015065, + 1.6278417270004866, + 1.7048445299988089, + 1.7743497830015258, + 1.8372234419985034, + 1.9273204279998026, + 2.023293363999983, + 2.1071446030000516, + 2.1792657529986172, + 2.24321523099934, + 2.3032377429990447, + 2.3593880660009745, + 2.4131707529995765, + 2.4704045440012123, + 2.540518156998587, + 2.6005126970012498, + 2.6546997090008517, + 2.7124728730013885, + 2.7762280230017495, + 2.834466981999867, + 2.9032788639997307, + 2.972037489998911, + 3.0375668330016197, + 3.102877024000918, + 3.157801711000502, + 3.216467244001251, + 3.2836657570005627, + 3.3380364600016037, + 3.3966619390012056, + 3.456425663000118, + 3.514862075000565, + 3.5801913409995905, + 3.654413378000754, + 3.7402248619982856, + 3.794846173001133, + 3.8551453670006595, + 3.9186142699982156, + 3.9729239039988897, + 4.026286328000424, + 4.0858286440015945, + 4.1460728220008605, + 4.204158781998558, + 4.273162026998762, + 4.330027088999486, + 4.394295468999189, + 4.480918814999313, + 4.567039206998743, + 4.660596026998974, + 4.750823952999781, + 4.835427008001716, + 4.9050646089999645, + 4.990383000000293, + 5.0764364909991855, + 5.1431991110002855, + 5.1947066339998855, + 5.2598025769984815, + 5.332010633999744, + 5.406793338999705, + 5.470574866001698, + 5.536448396000196, + 5.606428879000305, + 5.668005352999899, + 5.7330712789989775, + 5.802274197998486, + 5.855196625001554, + 5.919124782998551, + 6.001542509999126, + 6.093237138000404, + 6.184633818000293, + 6.261632908001047, + 6.324719503001688, + 6.398743402998662, + 6.488224345001072, + 6.585999163999077, + 6.6641599950016825, + 6.728273286000331, + 6.800041725000483, + 6.865393887001119, + 6.932564941998862, + 7.001154287001555, + 7.060602680001466, + 7.116336202998355, + 7.174073671001679, + 7.234201187999133, + 7.327782278000086 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.87352999849827, + "p95": 91.69462800127803, + "max": 151.6723959975934 + } + }, + { + "start": 30043.14599841, + "first": 30043.232206276, + "last": 30050.474961516, + "end": 30050.475069516, + "elapsed_s": 7.329071105999901, + "chunk_offsets_s": [ + 0.08620786599931307, + 0.40364420100013376, + 0.48154147600143915, + 0.5709678640014317, + 0.6398555859996122, + 0.6940972359989246, + 0.7485488380007155, + 0.8121852580006816, + 0.8773494640008721, + 0.9646786519988382, + 1.0375345829997968, + 1.1250277459985227, + 1.2069058000015502, + 1.2814997729983588, + 1.3455279929985409, + 1.4112927979986125, + 1.4877915229990322, + 1.563410409999051, + 1.628227158998925, + 1.7051148879982065, + 1.7745602129980398, + 1.8377352280003834, + 1.927553549001459, + 2.0236364209995372, + 2.10713674499857, + 2.1793585019986494, + 2.243502871999226, + 2.3034549260009953, + 2.360078148998582, + 2.4133879199980584, + 2.4709126480011037, + 2.5406614569983503, + 2.600915683000494, + 2.6549266209985944, + 2.712858655999298, + 2.776362201999291, + 2.8346267330016417, + 2.9036787769982766, + 2.971895788999973, + 3.0374376779982413, + 3.103089757998532, + 3.15794465900035, + 3.2166487100002996, + 3.2839420839991362, + 3.3380668159989, + 3.396780099999887, + 3.4566395969995938, + 3.515213196998957, + 3.5802824729980784, + 3.6546345770002517, + 3.739969302998361, + 3.7951194919987756, + 3.8555436479982745, + 3.9188710269991134, + 3.9728236330010986, + 4.026866923999478, + 4.085960630000045, + 4.14658052699815, + 4.204427092001424, + 4.273088463000022, + 4.330305351999414, + 4.394785986998613, + 4.480880023998907, + 4.567131891999452, + 4.660735294000915, + 4.75111637900045, + 4.835810295000556, + 4.905413746000704, + 4.99069459599923, + 5.077034720998199, + 5.143860566000512, + 5.195218067001406, + 5.259659242998168, + 5.332606608000788, + 5.407304197000485, + 5.470944263001002, + 5.536782939001569, + 5.606812500998785, + 5.668114984000567, + 5.733379002998845, + 5.802574352001102, + 5.855510317000153, + 5.919127294000646, + 6.0018620759983605, + 6.09350091899978, + 6.184695042000385, + 6.261831304000225, + 6.325209013000858, + 6.399073881999357, + 6.488805165001395, + 6.586393011999462, + 6.664716748000501, + 6.728874205000466, + 6.800596637000126, + 6.865926348000357, + 6.932790204999037, + 7.001654935000261, + 7.060980221998761, + 7.117506949998642, + 7.174624357998255, + 7.2348668049999105, + 7.328963105999719 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.76480500007165, + "p95": 91.63884300141945, + "max": 317.4363350008207 + } + }, + { + "start": 30043.1458959, + "first": 30043.481906973, + "last": 30050.475681431, + "end": 30050.475902919, + "elapsed_s": 7.330007018998003, + "chunk_offsets_s": [ + 0.33601107299909927, + 0.40386986399971647, + 0.48171367599934456, + 0.5709624869996333, + 0.6400565740004822, + 0.6940879699977813, + 0.748817498999415, + 0.8127744609992078, + 0.8774017910000111, + 0.9650236439993023, + 1.0379677319979237, + 1.125194920998183, + 1.2070820169974468, + 1.2816834779987403, + 1.3457132349976746, + 1.41158078999797, + 1.4880567759973928, + 1.563974649998272, + 1.6283664099973976, + 1.7053209650002827, + 1.7745958499981498, + 1.8378190309995261, + 1.9277344709989848, + 2.023571786998218, + 2.1071868799990625, + 2.179405611997936, + 2.243539899998723, + 2.303977720999683, + 2.360135757000535, + 2.4139272440006607, + 2.4711696129998018, + 2.5408238799973333, + 2.601095947997237, + 2.6550897319975775, + 2.712892436000402, + 2.776142776998313, + 2.834543519998988, + 2.9037310559979232, + 2.9724504929981777, + 3.03747616200053, + 3.103502600999491, + 3.157966822000162, + 3.216693372000009, + 3.2840999139989435, + 3.338119349999033, + 3.3970678039986524, + 3.4565095619982458, + 3.5154759859979094, + 3.5803211179991195, + 3.6545322889978706, + 3.7404282169991347, + 3.79528335500072, + 3.855590757997561, + 3.9189008219982497, + 3.973319751999952, + 4.027065959999163, + 4.086157089997869, + 4.146400835998065, + 4.20476227600011, + 4.273523949999799, + 4.330525671997748, + 4.394795635998889, + 4.480898761998105, + 4.567428317997837, + 4.660974687998532, + 4.7514674849990115, + 4.835852299998805, + 4.905446421998931, + 4.990742617999786, + 5.077355934998195, + 5.143909387999884, + 5.195366935997299, + 5.260277139997925, + 5.332643443998677, + 5.407285809000314, + 5.470994733997941, + 5.536783113999263, + 5.606719591000001, + 5.668076976999146, + 5.733524718998524, + 5.802742903997569, + 5.855651600999408, + 5.919746679999662, + 6.00253977399916, + 6.093537419998029, + 6.184702993999963, + 6.262024626998027, + 6.3252187889993365, + 6.399331213997357, + 6.489078483999037, + 6.586223610000161, + 6.664768353999534, + 6.728873804000614, + 6.800518304000434, + 6.8659816029976355, + 6.93281039899739, + 7.001542612997582, + 7.06097066700022, + 7.117782957997406, + 7.174779979999585, + 7.23466392499904, + 7.329785530997469 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.8675550002954, + "p95": 90.99764599886839, + "max": 97.14512600112357 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 141.0411861859606, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.320803457998409, + "end_to_end_tps": 139.87535738050988, + "timing_valid": true, + "steady_tps": 145.26630502139565, + "per_stream_tps": [ + 35.3, + 36.5, + 36.1, + 35.7 + ], + "ttft_s": [ + 0.09, + 0.34, + 0.26, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08832477599935373, + "decode_tps": 35.26029654649015, + "prefill_tps": null, + "itl_mean_ms": 28.360510203920594, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.57325400054106, + "round_gap_p50_ms": 65.21554500068305, + "round_gap_p95_ms": 92.5213509981404, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.34094902900324087, + "decode_tps": 36.543092578733855, + "prefill_tps": null, + "itl_mean_ms": 27.364952701949125, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.16534800239606, + "round_gap_p50_ms": 65.34067400207277, + "round_gap_p95_ms": 90.7657179996022, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2569192480004858, + "decode_tps": 36.105551513009026, + "prefill_tps": null, + "itl_mean_ms": 27.696571803914825, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.80110799832619, + "round_gap_p50_ms": 65.40663100167876, + "round_gap_p95_ms": 92.17582000201219, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17256755600101314, + "decode_tps": 35.677160670498324, + "prefill_tps": null, + "itl_mean_ms": 28.029136321571308, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.70698100104346, + "round_gap_p50_ms": 65.55249299708521, + "round_gap_p95_ms": 92.27755399842863, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.862792757838964, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.47090299928095, + "round_gap_p50_ms": 65.34067400207277, + "round_gap_p95_ms": 92.27755399842863, + "round_gap_mean_ms": 70.34665498266273, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30050.478563857, + "first": 30050.566888633, + "last": 30057.798818735, + "end": 30057.798999883, + "elapsed_s": 7.320436026002426, + "chunk_offsets_s": [ + 0.08832477599935373, + 0.410084442999505, + 0.487294913000369, + 0.5788039729995944, + 0.6501620369999728, + 0.7057430679997196, + 0.7618569619990012, + 0.825267571999575, + 0.8878745430010895, + 0.9694641290006984, + 1.0395349610007543, + 1.1262635349994525, + 1.2085554639998008, + 1.2834240660013165, + 1.346036009999807, + 1.4125564599999052, + 1.487471477001236, + 1.5683130910001637, + 1.6371156740024162, + 1.7167740229997435, + 1.7864586260002397, + 1.8507571449990792, + 1.9373303989996202, + 2.030142153002089, + 2.114095847999124, + 2.1882263150000654, + 2.251918502999615, + 2.3108400119999715, + 2.364013803002308, + 2.4166615630019805, + 2.4746911080001155, + 2.5436023010006465, + 2.6073655220025103, + 2.6650512230007735, + 2.7214143730016076, + 2.78424788600023, + 2.840358166002261, + 2.9025495349997072, + 2.9665764599994873, + 3.031606239001121, + 3.097449307999341, + 3.1533649990014965, + 3.210985106001317, + 3.2797423400006664, + 3.33435203100089, + 3.3933406569994986, + 3.4536356249991513, + 3.5128931239996746, + 3.5831286950015055, + 3.661714934001793, + 3.7468071780022, + 3.8030235309997806, + 3.860846892999689, + 3.9199387840017152, + 3.9709196260009776, + 4.023228140998981, + 4.0808512500007055, + 4.139365597002325, + 4.198632921001263, + 4.267974422000407, + 4.325295209000615, + 4.39031851999971, + 4.4774210329997, + 4.563863191000564, + 4.663035692999983, + 4.753174249999574, + 4.838006596000923, + 4.901881525001954, + 4.983181398001761, + 5.06974395799989, + 5.139550634001353, + 5.189981765000994, + 5.255104595002194, + 5.329556198001228, + 5.40573558899996, + 5.468870082000649, + 5.534085627001332, + 5.613060052000947, + 5.6758620780019555, + 5.737742155000888, + 5.807355842000106, + 5.860266955001862, + 5.9179090750003525, + 5.99896133600123, + 6.09148268699937, + 6.180951156999072, + 6.255587801002548, + 6.319284435001464, + 6.39328410500093, + 6.482525562001683, + 6.581576438999036, + 6.663476885001728, + 6.727846953999688, + 6.7965389320015674, + 6.862302445999376, + 6.920681645999139, + 6.987163813999359, + 7.047970425999665, + 7.1037688800024625, + 7.162361841001257, + 7.2249603910022415, + 7.320254877999105 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.21554500068305, + "p95": 92.5213509981404, + "max": 321.75966700015124 + } + }, + { + "start": 30050.478831577, + "first": 30050.819780606, + "last": 30057.797843545, + "end": 30057.797919909, + "elapsed_s": 7.3190883320021385, + "chunk_offsets_s": [ + 0.34094902900324087, + 0.41004366900233435, + 0.4874807160013006, + 0.5782464340009028, + 0.6502180930001487, + 0.7056776180033921, + 0.7615342730023258, + 0.8246442790004949, + 0.8877189360027842, + 0.9694158650017926, + 1.0400999250014138, + 1.1260551670020504, + 1.2084090260032099, + 1.282929288001469, + 1.3456655549998686, + 1.4110062290019414, + 1.4872596690001956, + 1.5686849860030634, + 1.6366887460026192, + 1.7164472220028983, + 1.7858808280034282, + 1.8505696130014258, + 1.93785598100294, + 2.0296928380012105, + 2.113914797002508, + 2.1878066370009037, + 2.251152565000666, + 2.310615147001954, + 2.363822927000001, + 2.4164520280028228, + 2.4742862950006383, + 2.5436115830016206, + 2.606827890002023, + 2.6648611950004124, + 2.720973187002528, + 2.7839433930021187, + 2.8401523920001637, + 2.902339408003172, + 2.9664223409999977, + 3.0313658990016847, + 3.0972210820000328, + 3.1530396540001675, + 3.211235768001643, + 3.279426389002765, + 3.334041888003412, + 3.393781012000545, + 3.453406567001366, + 3.5127323000015167, + 3.5826276920015516, + 3.66138744500131, + 3.7466210070015222, + 3.8031682890032243, + 3.8609312910011795, + 3.9195763770003396, + 3.970601050001278, + 4.0233955750009045, + 4.080808636001166, + 4.139266974001657, + 4.198428315001365, + 4.267815278002672, + 4.325306859002012, + 4.390134365003178, + 4.4776353540000855, + 4.563800702002482, + 4.663214697000512, + 4.752975261002575, + 4.837862878001033, + 4.900988142002461, + 4.983201386003202, + 5.069637590000639, + 5.139416054000321, + 5.189965411002049, + 5.255062588999863, + 5.329486668000754, + 5.405597745000705, + 5.4686403360028635, + 5.53399828700276, + 5.613027103001514, + 5.675361266003165, + 5.73764207700151, + 5.807211949002522, + 5.860064781001711, + 5.917724552000436, + 5.997762050003075, + 6.090137619001325, + 6.1806416060026095, + 6.255527824003366, + 6.318574696000724, + 6.393089820001478, + 6.48193412800174, + 6.581420512000477, + 6.663565411003219, + 6.727265666002495, + 6.795987488003448, + 6.862242485000024, + 6.92065341900161, + 6.987151045002975, + 7.047507316001429, + 7.102980712999852, + 7.162233918999846, + 7.2248700750024, + 7.319011968000268 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.34067400207277, + "p95": 90.7657179996022, + "max": 99.48638399873744 + } + }, + { + "start": 30050.478701557, + "first": 30050.735620805, + "last": 30057.798246615, + "end": 30057.799274453, + "elapsed_s": 7.320572895998339, + "chunk_offsets_s": [ + 0.2569192480004858, + 0.40983224600131507, + 0.4869570390001172, + 0.5787423479996505, + 0.6504461440017621, + 0.7056563990008726, + 0.7617725169984624, + 0.8250155669993546, + 0.887976719001017, + 0.9689453639985004, + 1.0399570410008891, + 1.126328296999418, + 1.208332950998738, + 1.2829820970000583, + 1.3456085229991004, + 1.4110469070001272, + 1.4877991739995196, + 1.5684416389995022, + 1.637113993998355, + 1.716796250999323, + 1.7862206229983713, + 1.8511655580005026, + 1.9380794229982712, + 2.0302552430002834, + 2.114175045000593, + 2.187601502999314, + 2.251827529998991, + 2.311146763000579, + 2.364165506998688, + 2.4169433339993702, + 2.474286231001315, + 2.543910923999647, + 2.6073777489982604, + 2.6653104149991123, + 2.7210419660004845, + 2.784242429999722, + 2.8404799769996316, + 2.902689892998751, + 2.9663175259993295, + 3.031431846000487, + 3.0975531170006434, + 3.1537641319991963, + 3.2110811689999537, + 3.27980948700133, + 3.33445539099921, + 3.393353251998633, + 3.4536587170005077, + 3.5129362670013506, + 3.5831299439996656, + 3.6621758680012135, + 3.746510765999119, + 3.803095862000191, + 3.860834076000174, + 3.9197528199983935, + 3.9708421750001435, + 4.023294456001167, + 4.081046912000602, + 4.139495329000056, + 4.198921125000197, + 4.268039375998342, + 4.325093739000295, + 4.390425288998813, + 4.477403783999762, + 4.563861111000733, + 4.663400229001127, + 4.752961386999232, + 4.838029464001011, + 4.901034629001515, + 4.983233214999927, + 5.070034322998254, + 5.139649674001703, + 5.19031770500078, + 5.255419140001322, + 5.329708141998708, + 5.405776971998421, + 5.468615787998715, + 5.534022419000394, + 5.612809406000451, + 5.675226798000949, + 5.737354880999192, + 5.807407835000049, + 5.8604804599999625, + 5.917586019000737, + 5.998078770000575, + 6.091252701000485, + 6.180847814001027, + 6.255773942000815, + 6.318923820999771, + 6.39335000400024, + 6.482673040998634, + 6.581509341998753, + 6.663510203001351, + 6.727578930000163, + 6.796332404999703, + 6.862427857999137, + 6.920896815998276, + 6.987080490998778, + 7.047878492998279, + 7.103316651999194, + 7.162562737001281, + 7.225100062001729, + 7.319545057998766 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.40663100167876, + "p95": 92.17582000201219, + "max": 152.91299800082925 + } + }, + { + "start": 30050.478470995, + "first": 30050.651038551, + "last": 30057.798468313, + "end": 30057.798535123, + "elapsed_s": 7.320064128001832, + "chunk_offsets_s": [ + 0.17256755600101314, + 0.409973257999809, + 0.48750761699920986, + 0.5787312419997761, + 0.6504578089989081, + 0.7057528289988113, + 0.7618175960014923, + 0.8249096789986652, + 0.887895779000246, + 0.9682725120001123, + 1.039774420998583, + 1.126496673001384, + 1.2084716280005523, + 1.2831494970014319, + 1.345792005999101, + 1.4124488759989617, + 1.4881973609990382, + 1.5687521969994123, + 1.6372833310015267, + 1.717075347998616, + 1.7867288909983472, + 1.8512089399991964, + 1.9376798429984774, + 2.030377516999579, + 2.1145274010013964, + 2.187768790998234, + 2.251794212999812, + 2.311146121999627, + 2.364328027000738, + 2.4172937699986505, + 2.4749401369990665, + 2.543747075000283, + 2.6073663710012625, + 2.6653094700013753, + 2.720992197999294, + 2.7845955709999544, + 2.8408042170012777, + 2.9028171280006063, + 2.966335095999966, + 3.0314289320012904, + 3.097958809001284, + 3.153777429000911, + 3.2115165940012957, + 3.2799890499991307, + 3.3345798250011285, + 3.3944222919999447, + 3.45407178700043, + 3.513278366001032, + 3.5828570859994215, + 3.6620994879995123, + 3.7468139630000223, + 3.8032314980009687, + 3.861366059998545, + 3.9198103320013615, + 3.9708341890000156, + 4.0238255110016325, + 4.08145231600065, + 4.139930754001398, + 4.199093887000345, + 4.268378370001301, + 4.3255862449987035, + 4.390776877000462, + 4.478195807001612, + 4.564298984001653, + 4.663385473999369, + 4.753583032001188, + 4.838000019000901, + 4.9017224599992915, + 4.983368163000705, + 5.070075144001748, + 5.139939988999686, + 5.19062402200143, + 5.255844418999914, + 5.330029262000608, + 5.4060692400016706, + 5.468777523001336, + 5.534330015998421, + 5.613089639999089, + 5.675203416998556, + 5.737372850999236, + 5.807819544999802, + 5.860805275999155, + 5.917660860999604, + 5.998408659001143, + 6.090686212999572, + 6.180837635001808, + 6.256078466998588, + 6.318821611999738, + 6.39383279699905, + 6.482697988001746, + 6.581637152001349, + 6.663815751999209, + 6.727494692000619, + 6.796231764001277, + 6.86207104999994, + 6.921254421999038, + 6.987190233998263, + 7.047707681998872, + 7.103758445999119, + 7.162704485999711, + 7.224938227998791, + 7.319997318001697 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.55249299708521, + "p95": 92.27755399842863, + "max": 237.40570199879585 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.5, + "per_stream_tps": 33.6, + "ttft_s_max": 0.44, + "request": { + "ttft_s": 0.255, + "decode_tps": 33.646, + "prefill_tps": null, + "itl_mean_ms": 29.721, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.195, + "round_gap_p50_ms": 69.174, + "round_gap_p95_ms": 98.095, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 29.724, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.049, + "round_gap_p50_ms": 69.124, + "round_gap_p95_ms": 98.318, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 163.2863410479541, + "makespan_s": 7.8389900330003, + "steady_tps": 171.0, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.3879236118382, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.842473646000144, + "end_to_end_tps": 163.21380954245626, + "timing_valid": true, + "steady_tps": 171.02424862865345, + "per_stream_tps": [ + 34.4, + 33.2, + 34.1, + 33.6, + 32.9 + ], + "ttft_s": [ + 0.44, + 0.17, + 0.36, + 0.26, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.4373202200004016, + "decode_tps": 34.437403140416606, + "prefill_tps": null, + "itl_mean_ms": 29.03819419607673, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.38324100078898, + "round_gap_p50_ms": 68.80260899924906, + "round_gap_p95_ms": 95.68261700042058, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.168165417999262, + "decode_tps": 33.23255898228829, + "prefill_tps": null, + "itl_mean_ms": 30.090971945102474, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.63113600172801, + "round_gap_p50_ms": 69.12394000028144, + "round_gap_p95_ms": 97.52385100000538, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3555182939999213, + "decode_tps": 34.064497610761755, + "prefill_tps": null, + "itl_mean_ms": 29.35607656471285, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.3906829993939, + "round_gap_p50_ms": 69.3292299984023, + "round_gap_p95_ms": 97.52020299856667, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2625651990019833, + "decode_tps": 33.64591644506012, + "prefill_tps": null, + "itl_mean_ms": 29.721288811760687, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.55341199968825, + "round_gap_p50_ms": 68.99403199713561, + "round_gap_p95_ms": 97.77080200001365, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08617479000167805, + "decode_tps": 32.88171566149464, + "prefill_tps": null, + "itl_mean_ms": 30.41203841960797, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.8879910024989, + "round_gap_p50_ms": 69.20593600079883, + "round_gap_p95_ms": 97.00306499871658, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.723713987452143, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.43050300210598, + "round_gap_p50_ms": 69.12394000028144, + "round_gap_p95_ms": 97.66009699887945, + "round_gap_mean_ms": 75.04502046336927, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30057.80054116, + "first": 30058.23786138, + "last": 30065.6426009, + "end": 30065.642740282, + "elapsed_s": 7.842199122002057, + "chunk_offsets_s": [ + 0.4373202200004016, + 0.5070358410011977, + 0.5899302550024004, + 0.6841954720002832, + 0.7544421990023693, + 0.8125091030015028, + 0.8683852230024058, + 0.933341329000541, + 0.9992887000007613, + 1.0842329260012775, + 1.1604401549993781, + 1.255571712001256, + 1.3457386310001311, + 1.4271474630004377, + 1.4922412340019946, + 1.559067848000268, + 1.6414779549995728, + 1.7234427600014897, + 1.789317516002484, + 1.8703430570021737, + 1.9468877460021758, + 2.0131637810009124, + 2.106285008001578, + 2.205491771001107, + 2.3011743880015274, + 2.3821909399994183, + 2.450524679999944, + 2.512556419002067, + 2.568271691001428, + 2.621180384001491, + 2.68028104500263, + 2.754644663000363, + 2.8192857980029657, + 2.8763113540007907, + 2.936639853000088, + 3.004131106001296, + 3.0645701570028905, + 3.132317433002754, + 3.2026662660027796, + 3.2744925600018178, + 3.3510185459999775, + 3.4083206770010293, + 3.467670743000781, + 3.5398616070015123, + 3.5962825770002382, + 3.655470025001705, + 3.7200357180008723, + 3.7808897140021145, + 3.8472019670007285, + 3.925194677001855, + 4.017730277999362, + 4.077994012001, + 4.139928423002857, + 4.2050718310019874, + 4.266997393002384, + 4.324167012000544, + 4.386930256001506, + 4.4512078810003, + 4.512295248001465, + 4.587658429001749, + 4.647670812002616, + 4.714064399002382, + 4.8062754920028965, + 4.895509160000074, + 4.998994550001953, + 5.0929053430008935, + 5.1852885840016825, + 5.25686449800196, + 5.348580273999687, + 5.438472756999545, + 5.513241523000033, + 5.567660010001418, + 5.635727582000982, + 5.715611471001466, + 5.793509975999768, + 5.860975053001312, + 5.927826930001174, + 6.006377101999533, + 6.070878138001717, + 6.139680747000966, + 6.214857400002074, + 6.2770426950010005, + 6.3418988270022965, + 6.4263186150019465, + 6.523978712000826, + 6.619100809002703, + 6.70408995100297, + 6.769702298002812, + 6.848747111002012, + 6.9408546699996805, + 7.045866488002503, + 7.129753059001814, + 7.198377442000492, + 7.277726223001082, + 7.350586923999799, + 7.4124254010021104, + 7.483896186000493, + 7.548078586001793, + 7.606392363002669, + 7.667264689000149, + 7.734399987999495, + 7.842059739999968 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.80260899924906, + "p95": 95.68261700042058, + "max": 107.65975200047251 + } + }, + { + "start": 30057.800683757, + "first": 30057.968849175, + "last": 30065.642047021, + "end": 30065.642135403, + "elapsed_s": 7.8414516459997685, + "chunk_offsets_s": [ + 0.168165417999262, + 0.5067405170011625, + 0.5900435210023716, + 0.6846100490001845, + 0.7539982279995456, + 0.8120818710012827, + 0.8679904750024434, + 0.9332736559990735, + 0.9985465710014978, + 1.0838507400003436, + 1.1600961990006908, + 1.2552692030003527, + 1.3457864160009194, + 1.4272151069999381, + 1.4921685680019436, + 1.559357509999245, + 1.6411567380018823, + 1.723507315000461, + 1.7893975290025992, + 1.870075671999075, + 1.9460395999994944, + 2.0128719120002643, + 2.1055030480019923, + 2.2050159070022346, + 2.301102617999277, + 2.3817111550015397, + 2.45003740500033, + 2.512205005001306, + 2.56757790499978, + 2.6206892379996134, + 2.6798960430023726, + 2.754011174001789, + 2.819523308000498, + 2.8758139820020006, + 2.935982311999396, + 3.00306050799918, + 3.064055326001835, + 3.131836687000032, + 3.202176254999358, + 3.2742151900019962, + 3.3505808800000523, + 3.408021800001734, + 3.4675903000024846, + 3.539572555000632, + 3.5960743530013133, + 3.655570393999369, + 3.7198488419999194, + 3.780658799001685, + 3.846880910001346, + 3.9248783250004635, + 4.017513621001854, + 4.0778080800009775, + 4.1399384579999605, + 4.205026578001707, + 4.267264108002564, + 4.3240727900010825, + 4.387060389999533, + 4.450764421002532, + 4.512183999999252, + 4.587635898998997, + 4.647402275000786, + 4.713718635000987, + 4.805569895001099, + 4.895541598001728, + 4.999001079999289, + 5.09304211700146, + 5.185442560999945, + 5.256576198000403, + 5.348515065001266, + 5.438396249999641, + 5.512949847001437, + 5.567331720001675, + 5.6361087469995255, + 5.715524449999066, + 5.7934758119990875, + 5.860027850001643, + 5.927561738000804, + 6.005908054001338, + 6.070938324999588, + 6.1400622649998695, + 6.214639239002281, + 6.277765033999458, + 6.341109632001462, + 6.4260937010003545, + 6.52361755200036, + 6.618410702001711, + 6.703295523002453, + 6.769513277002261, + 6.849123620002501, + 6.940642558001855, + 7.04554471099982, + 7.129129842000111, + 7.198167122001905, + 7.277325842002028, + 7.350898317999963, + 7.412527083000896, + 7.483559685999353, + 7.547538071001327, + 7.6057745550024265, + 7.666620396001235, + 7.73383350000222, + 7.8413632640003925 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.12394000028144, + "p95": 97.52385100000538, + "max": 338.5750990019005 + } + }, + { + "start": 30057.801072937, + "first": 30058.156591231, + "last": 30065.642390755, + "end": 30065.642426889, + "elapsed_s": 7.841353952000645, + "chunk_offsets_s": [ + 0.3555182939999213, + 0.5067662319997908, + 0.5894777380017331, + 0.6842619790004392, + 0.7541418660002819, + 0.8120741249986168, + 0.8679128149997268, + 0.932784876000369, + 0.9988315750015317, + 1.0833805389993358, + 1.1600345260012546, + 1.2552423349989112, + 1.3456691020001017, + 1.4267128210012743, + 1.4919776429997, + 1.5584152529991115, + 1.6411439520015847, + 1.72284098900127, + 1.7893566749989986, + 1.870491112000309, + 1.9459545470017474, + 2.0134650930012867, + 2.1052874790002534, + 2.2050242600016645, + 2.3010674610013666, + 2.381857434000267, + 2.450031308999314, + 2.512545426001452, + 2.568224148999434, + 2.6205287010016036, + 2.6795501179985877, + 2.7538934760013944, + 2.818794122998952, + 2.8753000139986398, + 2.935669480000797, + 3.0028888500019093, + 3.0638596640019387, + 3.131610380998609, + 3.2016867230013304, + 3.2739061989996117, + 3.350378704999457, + 3.407716616999096, + 3.4672709229998873, + 3.5394040329993004, + 3.595640767001896, + 3.6550179080004455, + 3.719367728001089, + 3.78032825199989, + 3.846982189999835, + 3.924942383000598, + 4.017333065999992, + 4.077537959001347, + 4.139509352000459, + 4.2045851170005335, + 4.266829913001857, + 4.323864838999725, + 4.386161753998749, + 4.450439139000082, + 4.511699701000907, + 4.587037230001442, + 4.647098628000094, + 4.713284535999264, + 4.805354870000883, + 4.8950986659983755, + 4.998545954000292, + 5.092420052998932, + 5.185124368999823, + 5.25651460500012, + 5.347944321001705, + 5.438444082999922, + 5.512876700999186, + 5.567439146001561, + 5.635096860998601, + 5.71529409499999, + 5.793131182999787, + 5.859751375999622, + 5.927120790001936, + 6.005476947000716, + 6.070728884998971, + 6.138971582000522, + 6.214722837001318, + 6.276387251000415, + 6.340581838001526, + 6.426232940000773, + 6.52375314299934, + 6.618307710999943, + 6.703027690000454, + 6.769019056999241, + 6.848275871998339, + 6.940538974999072, + 7.045444169001712, + 7.12880812900039, + 7.1981373589987925, + 7.276809825998498, + 7.35015604200089, + 7.41218978300094, + 7.48341740899923, + 7.547088145001908, + 7.605500336998375, + 7.666565445000742, + 7.7336766439984785, + 7.8413178180016985 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.3292299984023, + "p95": 97.52020299856667, + "max": 151.24793799986946 + } + }, + { + "start": 30057.801184186, + "first": 30058.063749385, + "last": 30065.642678032, + "end": 30065.642932697, + "elapsed_s": 7.841748511000333, + "chunk_offsets_s": [ + 0.2625651990019833, + 0.5065398940023442, + 0.5894994539994514, + 0.6831797330014524, + 0.7541843200015137, + 0.8120929120013898, + 0.8679311220003001, + 0.9329928440020012, + 0.9984969630022533, + 1.0834042070018768, + 1.1602539520026767, + 1.2548595879998174, + 1.3456679020018782, + 1.4259011600006488, + 1.4920457810003427, + 1.5585002100015117, + 1.6407328170025721, + 1.722766369002784, + 1.7890504999995755, + 1.8699824899995292, + 1.9460620999998355, + 2.0128556540003046, + 2.1055375340001774, + 2.2047452250008064, + 2.3010542590018304, + 2.382471721000911, + 2.4499648830023943, + 2.512552116000734, + 2.56787808000081, + 2.6200238070014166, + 2.679678825999872, + 2.7534150339997723, + 2.8189107819998753, + 2.8750881740015757, + 2.9357907150006213, + 3.0029501560020435, + 3.0640873320007813, + 3.1312513970005966, + 3.2017477890003647, + 3.273947838002641, + 3.350301541002409, + 3.407647662999807, + 3.467271403002087, + 3.5389837760012597, + 3.595611226002802, + 3.6547711980019812, + 3.7192997980018845, + 3.7802736520025064, + 3.8468098280027334, + 3.924729615002434, + 4.017157967002277, + 4.077560171001096, + 4.139521225999488, + 4.204560778001905, + 4.266299478000292, + 4.32391860700227, + 4.386333476999425, + 4.450296653001715, + 4.511710391001543, + 4.5872614569998404, + 4.647105605999968, + 4.713253267000255, + 4.805683770002361, + 4.8951131960020575, + 4.998145363999356, + 5.092139609001606, + 5.184693021001294, + 5.256308252999588, + 5.347679736001737, + 5.438189947002684, + 5.5124126600021555, + 5.567223977002868, + 5.635485738999705, + 5.715091584002948, + 5.792934593002428, + 5.8600150820020644, + 5.927150875002553, + 6.005464226000186, + 6.070669708002242, + 6.139663739999378, + 6.214389488999586, + 6.276990641999873, + 6.340922819999832, + 6.425799784999981, + 6.523570586999995, + 6.617837589001283, + 6.7031204250015435, + 6.769117969000945, + 6.848494642999867, + 6.940477317999466, + 7.045486192000681, + 7.129001088000223, + 7.197800218000339, + 7.277238214002864, + 7.350558443002228, + 7.411568662002537, + 7.483422067001811, + 7.547222583001712, + 7.605903474002844, + 7.66620553300163, + 7.733236239000689, + 7.841493846000958 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.99403199713561, + "p95": 97.77080200001365, + "max": 243.97469500036095 + } + }, + { + "start": 30057.800459051, + "first": 30057.886633841, + "last": 30065.641703638, + "end": 30065.641780883, + "elapsed_s": 7.841321832001995, + "chunk_offsets_s": [ + 0.08617479000167805, + 0.5067669519994524, + 0.5899435540013656, + 0.6837463150004623, + 0.7541080200026045, + 0.8121620260026248, + 0.8681591560016386, + 0.9332321130023047, + 0.9989682350023941, + 1.0838435700025002, + 1.1602274350007065, + 1.2555355950025842, + 1.3457097150021582, + 1.4264866260018607, + 1.4922180310022668, + 1.5596381290015415, + 1.64133806099926, + 1.7236962240021967, + 1.7892598910002562, + 1.8702445939998142, + 1.946202537001227, + 2.013020063001022, + 2.1067015690023254, + 2.205295997002395, + 2.3015366040017398, + 2.3820966860002954, + 2.4501850590022514, + 2.5122405300026003, + 2.5676022120023845, + 2.6206425730015326, + 2.6800468969995563, + 2.753878015999362, + 2.8191760370027623, + 2.87548146500194, + 2.9358826160023455, + 3.002995328999532, + 3.063991539002018, + 3.131759714000509, + 3.202565242001583, + 3.274937829999544, + 3.3503822720012977, + 3.4087549850009964, + 3.4675395550002577, + 3.5394694179994985, + 3.596099860002141, + 3.6553398450014356, + 3.719839960001991, + 3.780627800999355, + 3.846883197002171, + 3.924734989999706, + 4.017622981002205, + 4.077898637002363, + 4.139853788001346, + 4.204902716999641, + 4.26728553400244, + 4.324130446002528, + 4.386370440999599, + 4.450856259001739, + 4.51212061300248, + 4.587429368999437, + 4.647377118002623, + 4.713522155001556, + 4.8054141419997904, + 4.895389957000589, + 4.9986794290016405, + 5.092787740002677, + 5.185188023999217, + 5.256898982999701, + 5.348999892001302, + 5.43812115000037, + 5.513711101000808, + 5.567464203999407, + 5.6354913850009325, + 5.716199227001198, + 5.794005925999954, + 5.860971068999788, + 5.928064600000653, + 6.005816709999635, + 6.070837875999132, + 6.140043811999931, + 6.21499617399968, + 6.277511393000168, + 6.341910669001663, + 6.426699299001484, + 6.523702364000201, + 6.618828622002184, + 6.704012099002284, + 6.769931758000894, + 6.849561512000946, + 6.941033530001732, + 7.045720689002337, + 7.12989543400181, + 7.198101334000967, + 7.277859347999765, + 7.3514768880013435, + 7.412059536000015, + 7.483413070000097, + 7.547452071001317, + 7.606315727000037, + 7.666338062001159, + 7.733615815999656, + 7.841244587001711 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.20593600079883, + "p95": 97.00306499871658, + "max": 420.5921619977744 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.47880214924442, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.8389900330003, + "end_to_end_tps": 163.2863410479541, + "timing_valid": true, + "steady_tps": 170.90950051387503, + "per_stream_tps": [ + 34.4, + 33.6, + 34.0, + 33.3, + 32.9 + ], + "ttft_s": [ + 0.43, + 0.25, + 0.34, + 0.17, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.42841263499940396, + "decode_tps": 34.411897394939096, + "prefill_tps": null, + "itl_mean_ms": 29.05971700784707, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.97558999949251, + "round_gap_p50_ms": 69.76709300215589, + "round_gap_p95_ms": 96.22359200147912, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25497169499794836, + "decode_tps": 33.632076619821945, + "prefill_tps": null, + "itl_mean_ms": 29.733519321570046, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.56370900038746, + "round_gap_p50_ms": 70.18231199981528, + "round_gap_p95_ms": 98.31806200236315, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3384930100000929, + "decode_tps": 34.00786962507834, + "prefill_tps": null, + "itl_mean_ms": 29.404958647058926, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.23708900026395, + "round_gap_p50_ms": 70.43364800119889, + "round_gap_p95_ms": 98.09548300108872, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17056126299939933, + "decode_tps": 33.26152081422261, + "prefill_tps": null, + "itl_mean_ms": 30.064770807845942, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.04934599983972, + "round_gap_p50_ms": 70.19435500114923, + "round_gap_p95_ms": 98.02687899718876, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08704179000051226, + "decode_tps": 32.895760429848885, + "prefill_tps": null, + "itl_mean_ms": 30.399054070585407, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.19549900080892, + "round_gap_p50_ms": 69.9616199999582, + "round_gap_p95_ms": 99.11399699922185, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.732403970981476, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.04934599983972, + "round_gap_p50_ms": 70.129806001205, + "round_gap_p95_ms": 98.31806200236315, + "round_gap_mean_ms": 75.06696052079482, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30065.64434332, + "first": 30066.072755955, + "last": 30073.482983792, + "end": 30073.483086128, + "elapsed_s": 7.838742807998642, + "chunk_offsets_s": [ + 0.42841263499940396, + 0.5018721779997577, + 0.58749209799862, + 0.6802069189980102, + 0.7510338840002078, + 0.8073305920006533, + 0.8650573660015652, + 0.9304050940008892, + 0.9956186400013394, + 1.0800269749997824, + 1.1553350430003775, + 1.2477158020010393, + 1.3319197700002405, + 1.4165699699988181, + 1.486337063000974, + 1.5552279369985627, + 1.6391844739991939, + 1.720640485000331, + 1.7882328490013606, + 1.8714654299983522, + 1.9499390780001704, + 2.0172134390013525, + 2.109189029000845, + 2.206878846998734, + 2.296067499999481, + 2.373942777998309, + 2.4496730569990177, + 2.5143150510011765, + 2.5709596880005847, + 2.6256447960004152, + 2.6870794620008382, + 2.759258772999601, + 2.824438357998588, + 2.881356974001392, + 2.9403937210008735, + 3.007990604000952, + 3.0701083069980086, + 3.138195684998209, + 3.208325490999414, + 3.2747175629992853, + 3.3458270150003955, + 3.406756194999616, + 3.4704246949986555, + 3.544868201999634, + 3.6022000369994203, + 3.661407996998605, + 3.7262963719986146, + 3.787596571000904, + 3.854800453998905, + 3.932429799999227, + 4.023227786998177, + 4.083244932000525, + 4.1426506050011085, + 4.204492926000967, + 4.258169275999535, + 4.313317608000943, + 4.375533828999323, + 4.446604706998187, + 4.509716208998725, + 4.583485889001167, + 4.644821984998998, + 4.7100601800011646, + 4.801138791000994, + 4.891093604001071, + 4.992595651998272, + 5.0875341319988365, + 5.180208762998518, + 5.24828454100134, + 5.336650954999641, + 5.4314891190006165, + 5.509120599999733, + 5.563462160000199, + 5.632024416001514, + 5.7102676590002375, + 5.790229772999737, + 5.857608750000509, + 5.923690284998884, + 6.000440904001152, + 6.064495311999053, + 6.131317062998278, + 6.207278452999162, + 6.2628140119995805, + 6.324171362000925, + 6.413528880999365, + 6.517010655999911, + 6.61323424800139, + 6.69821030499952, + 6.764962417000788, + 6.84493503900012, + 6.937000949001231, + 7.043433457998617, + 7.127537421998568, + 7.195215023999481, + 7.268646305998118, + 7.339926068998466, + 7.402901298999495, + 7.480534096001065, + 7.546424503001617, + 7.604429212999094, + 7.666474259000097, + 7.734508582998387, + 7.838640472000407 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.76709300215589, + "p95": 96.22359200147912, + "max": 106.43250899738632 + } + }, + { + "start": 30065.644631333, + "first": 30065.899603028, + "last": 30073.481650455, + "end": 30073.482399969, + "elapsed_s": 7.837768635999964, + "chunk_offsets_s": [ + 0.25497169499794836, + 0.5014384619971679, + 0.5869484279974131, + 0.6795121369978006, + 0.750818441996671, + 0.8064236159989377, + 0.8650073749995499, + 0.9304878749971977, + 0.9948746179979935, + 1.0800974029989447, + 1.154659271996934, + 1.2479094649970648, + 1.3315027929966163, + 1.4164588489984453, + 1.4866411609982606, + 1.5548593349994917, + 1.637910399997054, + 1.7204640099989774, + 1.7878006369974173, + 1.8714794169973175, + 1.9492481209999823, + 2.017120848999184, + 2.1085745959971973, + 2.2068926579995605, + 2.296274189997348, + 2.3738694229978137, + 2.4491646569986187, + 2.5134445289986616, + 2.5709116329999233, + 2.6255626549973385, + 2.6864865839997947, + 2.759960373998183, + 2.824339702998259, + 2.880955662996712, + 2.9407629009983793, + 3.0081129929967574, + 3.069337192999228, + 3.138305238997418, + 3.2085249479969207, + 3.274828477999108, + 3.3460642069985624, + 3.406265589997929, + 3.4699146619968815, + 3.5447409989974403, + 3.6019687529988005, + 3.6612356349978654, + 3.7259350509993965, + 3.7878699289976794, + 3.854311480998149, + 3.932265247000032, + 4.023262384998816, + 4.082503742996778, + 4.142239916000108, + 4.2043638179966365, + 4.258052810997469, + 4.312880738998501, + 4.3754322299973865, + 4.446605812998314, + 4.509363176999614, + 4.583542651998869, + 4.644492316998367, + 4.70954621099736, + 4.800905137999507, + 4.890879602997302, + 4.992405942997721, + 5.0874706670001615, + 5.180549635999341, + 5.24810459299988, + 5.3369190960002015, + 5.431254410999827, + 5.50877896199745, + 5.563771954999538, + 5.632769754000037, + 5.709680109997862, + 5.790002712998103, + 5.857151748998149, + 5.92371966799692, + 6.000693297999533, + 6.06412779799939, + 6.131502022999484, + 6.207148817997222, + 6.262392488999467, + 6.323190868999518, + 6.413136594997923, + 6.516772010996647, + 6.613306420997105, + 6.698098703996948, + 6.764910249999957, + 6.844069748000038, + 6.936785762998625, + 7.042990459998691, + 7.127058594000118, + 7.195276811999065, + 7.268260404998728, + 7.339820037999743, + 7.4029378669984, + 7.480096777999279, + 7.5459848969985615, + 7.604055025996786, + 7.66574833499908, + 7.734005782996974, + 7.83701912199831 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.18231199981528, + "p95": 98.31806200236315, + "max": 246.46676699921954 + } + }, + { + "start": 30065.645178828, + "first": 30065.983671838, + "last": 30073.481936293, + "end": 30073.482053608, + "elapsed_s": 7.836874780001381, + "chunk_offsets_s": [ + 0.3384930100000929, + 0.501229900000908, + 0.5865777610015357, + 0.679912265000894, + 0.7505710910008929, + 0.8059485810008482, + 0.8644079990008322, + 0.930154958001367, + 0.9946009109989973, + 1.0786342110004625, + 1.1548976300000504, + 1.2472448640000948, + 1.3305884870023874, + 1.415535693999118, + 1.4859693420003168, + 1.554603295000561, + 1.6375117610004963, + 1.7193427439997322, + 1.7873769370016817, + 1.8708801220018358, + 1.9486503779990016, + 2.016682875000697, + 2.1085356299990963, + 2.206631113000185, + 2.2954912339992006, + 2.3733758090020274, + 2.448991668999952, + 2.5133840880007483, + 2.570628420999128, + 2.6254198650021863, + 2.6861082680006803, + 2.758662326999911, + 2.8234626990015386, + 2.8805868689996714, + 2.9399263840023195, + 3.0072671779998927, + 3.069186050001008, + 3.137154047999502, + 3.207682300999295, + 3.273963588002516, + 3.3452996849991905, + 3.4059957070021483, + 3.46980739799983, + 3.5439743810020445, + 3.6017486230011855, + 3.6609099990018876, + 3.7255992220016196, + 3.7869681689990102, + 3.85354052600087, + 3.931766816000163, + 4.022530557002028, + 4.082494254002086, + 4.142373202001181, + 4.203884877999371, + 4.257728807999229, + 4.312189893000323, + 4.375079150002421, + 4.445849835999979, + 4.509142487000645, + 4.582731017999322, + 4.6440666660018906, + 4.7086099490006745, + 4.800441465002223, + 4.890975940001226, + 4.9922039750017575, + 5.087642823000351, + 5.179880697000044, + 5.247818916999677, + 5.336033595001936, + 5.430374014002155, + 5.508332507000887, + 5.562863666000339, + 5.632180156000686, + 5.709800274998997, + 5.789681367001322, + 5.8569464220017835, + 5.923069852000481, + 6.000007844999345, + 6.0636165169999, + 6.130736908999097, + 6.206846739001776, + 6.261764709001, + 6.322563057001389, + 6.412763880001876, + 6.516401985001721, + 6.612711238001793, + 6.697797504999471, + 6.7644853349993355, + 6.844197467002232, + 6.936434556002496, + 7.042855944000621, + 7.127017629001784, + 7.194643926999561, + 7.2682683279999765, + 7.338946058000147, + 7.402350668999134, + 7.4796016280015465, + 7.545813992001058, + 7.603757715001848, + 7.665531374001148, + 7.733889094000915, + 7.836757465000119 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.43364800119889, + "p95": 98.09548300108872, + "max": 162.73689000081504 + } + }, + { + "start": 30065.644491311, + "first": 30065.815052574, + "last": 30073.48156913, + "end": 30073.482674925, + "elapsed_s": 7.838183613999718, + "chunk_offsets_s": [ + 0.17056126299939933, + 0.5018702420020418, + 0.5871677110008022, + 0.6799682420023601, + 0.7511493910024001, + 0.8069954029997461, + 0.8651855310017709, + 0.9309016690021963, + 0.9952071190018614, + 1.079210702002456, + 1.1552691930010042, + 1.2473185390008439, + 1.3314020079997135, + 1.416378292000445, + 1.4867292060007458, + 1.5551828269999532, + 1.6387896920023195, + 1.7204237309997552, + 1.7878652710023744, + 1.87142300800042, + 1.949506018001557, + 2.0171357470026123, + 2.108815546002006, + 2.206842424999195, + 2.2959837920025166, + 2.373858476999885, + 2.4490305069994065, + 2.5139374080026755, + 2.57120350400146, + 2.6259134150022874, + 2.687088392001897, + 2.759923808000167, + 2.824391279002157, + 2.8811617760002264, + 2.9405453780018433, + 3.008055174999754, + 3.0699268550015404, + 3.1379574070015224, + 3.208642662000784, + 3.2744501439992746, + 3.34637216000192, + 3.4065605649993813, + 3.470127720000164, + 3.5449885740017635, + 3.6023109039997507, + 3.661219199999323, + 3.726213543002814, + 3.788084923002316, + 3.854331163001916, + 3.93250589300078, + 4.022984884002653, + 4.083285356002307, + 4.1428812660014955, + 4.204237941001338, + 4.257697872999415, + 4.313072257002204, + 4.375341622999258, + 4.446405107999453, + 4.5093213460022525, + 4.583618903001479, + 4.64447623399974, + 4.710150708000583, + 4.801262570999825, + 4.891145020999829, + 4.9926869789997, + 5.088279515999602, + 5.1807347210015, + 5.248065563002456, + 5.336639130000549, + 5.431145591999666, + 5.509209670999553, + 5.56373535700186, + 5.632516096000472, + 5.7103382800014515, + 5.790307033999852, + 5.85737418500139, + 5.9235899660016, + 6.000762173000112, + 6.064134215001104, + 6.131339948002278, + 6.208002268002019, + 6.262717229001282, + 6.324291414002801, + 6.413211229999433, + 6.51680972900067, + 6.612785216002521, + 6.698430341002677, + 6.7651334530019085, + 6.844972470000357, + 6.9371825470007025, + 7.043403374002082, + 7.1274445920025755, + 7.195468971000082, + 7.2690108780006994, + 7.339205233001849, + 7.4028368899998895, + 7.480532272002165, + 7.5463630220001505, + 7.604586935001862, + 7.666110584999842, + 7.734273090001807, + 7.837077819000115 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.19435500114923, + "p95": 98.02687899718876, + "max": 331.30897900264245 + } + }, + { + "start": 30065.644239527, + "first": 30065.731281317, + "last": 30073.483040105, + "end": 30073.48322956, + "elapsed_s": 7.8389900330003, + "chunk_offsets_s": [ + 0.08704179000051226, + 0.5020664170006057, + 0.586945723000099, + 0.6801790429999528, + 0.7509881010009849, + 0.8076299519998429, + 0.8650693359995785, + 0.9309904190013185, + 0.9953123840023181, + 1.080031505000079, + 1.1552711610020197, + 1.2476510080014123, + 1.3314197710024018, + 1.4165687700005947, + 1.4865303900005529, + 1.5558089100013603, + 1.6391102870002214, + 1.7201710350018402, + 1.7879894500001683, + 1.8712756730019464, + 1.9493786540006113, + 2.0171108150025248, + 2.1087351650021446, + 2.2078491620013665, + 2.296082335000392, + 2.3739202310025576, + 2.4491938130013295, + 2.514543056000548, + 2.570969034000882, + 2.6265433879998454, + 2.6867360310025106, + 2.759432609000214, + 2.824289326003054, + 2.881217624002602, + 2.9411196620021656, + 3.0082543110002007, + 3.069901587001368, + 3.1380215610006417, + 3.208839397000702, + 3.2746026330023597, + 3.346060365001904, + 3.407140593000804, + 3.470434393002506, + 3.5447353970012045, + 3.602205158000288, + 3.661335714001325, + 3.7262713119998807, + 3.7878036450019863, + 3.8543781460030004, + 3.9324901620020682, + 4.02340909600025, + 4.082730018002621, + 4.14259370000218, + 4.204318530002638, + 4.2578042580025794, + 4.312987315002829, + 4.3755330450003385, + 4.446507444001327, + 4.50969561400052, + 4.583533353001258, + 4.644536851003068, + 4.709671382002853, + 4.801116419002938, + 4.8911041660030605, + 4.993248541002686, + 5.087792333000834, + 5.180666596999799, + 5.24814677500035, + 5.337087874002464, + 5.4311652750002395, + 5.5090191760027665, + 5.563639964002505, + 5.632037346000288, + 5.710182750000968, + 5.790241375001642, + 5.857505821000814, + 5.923683356002584, + 6.001196561999677, + 6.064320436002163, + 6.132027210001979, + 6.20805369899972, + 6.262550465002278, + 6.324750560001121, + 6.413367367000319, + 6.517413634002878, + 6.612920341001882, + 6.698160377000022, + 6.765210473000479, + 6.844775989000482, + 6.936971488001291, + 7.043255061002128, + 7.127258593001898, + 7.196088591001171, + 7.269437587001448, + 7.339977774001454, + 7.402878447999683, + 7.480333903000428, + 7.5464643180021085, + 7.604315994001809, + 7.666261473001214, + 7.73428174700166, + 7.838800577999791 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.9616199999582, + "p95": 99.11399699922185, + "max": 415.02462700009346 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.82879901267688, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.821783258001233, + "end_to_end_tps": 163.64554702927032, + "timing_valid": true, + "steady_tps": 171.0601570092834, + "per_stream_tps": [ + 34.1, + 33.3, + 33.0, + 34.4, + 33.7 + ], + "ttft_s": [ + 0.33, + 0.17, + 0.09, + 0.42, + 0.25 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.33406845400168095, + "decode_tps": 34.06291476219884, + "prefill_tps": null, + "itl_mean_ms": 29.357440694116566, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.13010000166832, + "round_gap_p50_ms": 68.87481499870773, + "round_gap_p95_ms": 99.30271100165555, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1679861249976966, + "decode_tps": 33.32107029524969, + "prefill_tps": null, + "itl_mean_ms": 30.011040796086366, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.83561999816447, + "round_gap_p50_ms": 68.74845700076548, + "round_gap_p95_ms": 98.96143899823073, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08529396199810435, + "decode_tps": 32.96862598035009, + "prefill_tps": null, + "itl_mean_ms": 30.331867654903736, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.08033200047794, + "round_gap_p50_ms": 68.66184700265876, + "round_gap_p95_ms": 99.64602699983516, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.41781833600180107, + "decode_tps": 34.44567863716663, + "prefill_tps": null, + "itl_mean_ms": 29.03121783529059, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.57107799910591, + "round_gap_p50_ms": 68.63443900147104, + "round_gap_p95_ms": 98.18014399934327, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25129432399990037, + "decode_tps": 33.68970885590304, + "prefill_tps": null, + "itl_mean_ms": 29.6826548509867, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.92038299806882, + "round_gap_p50_ms": 69.17441700352356, + "round_gap_p95_ms": 99.47222299888381, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.682844366276793, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.88408899717615, + "round_gap_p50_ms": 68.78625299941632, + "round_gap_p95_ms": 99.48563299985835, + "round_gap_mean_ms": 74.94183478614437, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30073.484881348, + "first": 30073.818949802, + "last": 30081.305097179, + "end": 30081.305806978, + "elapsed_s": 7.82092563000333, + "chunk_offsets_s": [ + 0.33406845400168095, + 0.48669461000099545, + 0.5736627130027045, + 0.6731508110024151, + 0.7449000490014441, + 0.8035798910023004, + 0.8598918429997866, + 0.9258673870026541, + 0.9904942150023999, + 1.0748814020007558, + 1.1503080390029936, + 1.2421921280001698, + 1.3257042410004942, + 1.405780766002863, + 1.4716560880006, + 1.5393442660024448, + 1.6224947980008437, + 1.7097524020027777, + 1.7773506459998316, + 1.8605572510023194, + 1.9368770820001373, + 2.004422260000865, + 2.0966799779998837, + 2.1959826890015393, + 2.2838673559999734, + 2.3604405380028766, + 2.4278508330025943, + 2.4896087440029078, + 2.5461172020004597, + 2.602164497002377, + 2.6681663039998966, + 2.7451619490020676, + 2.808151982000709, + 2.865368204002152, + 2.924455822001619, + 2.990955727000255, + 3.051633381001011, + 3.117689149999933, + 3.187247046000266, + 3.2540886080023483, + 3.3254481550029595, + 3.384146137002972, + 3.4442533280016505, + 3.5178437780014065, + 3.574861963999865, + 3.63930785300181, + 3.708370342003036, + 3.769487603000016, + 3.836679448002542, + 3.9151074460023665, + 4.007943800999783, + 4.067080447002809, + 4.12858676399992, + 4.192284210002981, + 4.2458504360001825, + 4.3010162050013605, + 4.362362892999954, + 4.425253677000001, + 4.487807333000092, + 4.5641360160007025, + 4.629081331000634, + 4.698099319000903, + 4.789081749000616, + 4.878653370000393, + 4.979780499001208, + 5.074966231000872, + 5.166469371000858, + 5.235227315002703, + 5.3214438000031805, + 5.411476709003182, + 5.487457403000008, + 5.541952523002692, + 5.610824600000342, + 5.695831781002198, + 5.773211522002384, + 5.840246367002692, + 5.907364425002015, + 5.984932870000193, + 6.049865191002027, + 6.118195962000755, + 6.193361528999958, + 6.2494892710019485, + 6.308992568003305, + 6.394825735002087, + 6.492136775003019, + 6.590838911000901, + 6.6829690110025695, + 6.751572836001287, + 6.830848754001636, + 6.925044221003191, + 7.031207404001179, + 7.11518059700029, + 7.182294751000882, + 7.256337015001918, + 7.326371570001356, + 7.387484164002672, + 7.459224079000705, + 7.522923296000954, + 7.579993248000392, + 7.647224669002753, + 7.716099484001461, + 7.820215831001406 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.87481499870773, + "p95": 99.30271100165555, + "max": 152.6261559993145 + } + }, + { + "start": 30073.484543613, + "first": 30073.652529738, + "last": 30081.305345141, + "end": 30081.306224871, + "elapsed_s": 7.821681257999444, + "chunk_offsets_s": [ + 0.1679861249976966, + 0.4873299929968198, + 0.5738407259996166, + 0.6733263589994749, + 0.7456744789997174, + 0.8039653929990891, + 0.8603235769987805, + 0.9262907839984109, + 0.9906622579983377, + 1.0750775449996581, + 1.1501473409989558, + 1.2419355419988278, + 1.3259946559992386, + 1.4061770549997163, + 1.4720330929994816, + 1.5398843379989557, + 1.6229387589992257, + 1.7102618609969795, + 1.777836932997161, + 1.8609738949999155, + 1.937585980998847, + 2.0050915219981107, + 2.0974134579992096, + 2.1963748969974404, + 2.2846574049981427, + 2.361038875998929, + 2.428439936997165, + 2.490373364998959, + 2.5467875359972822, + 2.6029140909995476, + 2.669081861997256, + 2.7459066139999777, + 2.808787621997908, + 2.8657533549994696, + 2.9248573429977114, + 2.9909329389993218, + 3.051697871997021, + 3.118078206000064, + 3.18766477799727, + 3.254491280997172, + 3.325851067998883, + 3.3845783039978414, + 3.4447111310000764, + 3.5178378249984235, + 3.5752508919977117, + 3.6398529819998657, + 3.708654435999051, + 3.769871312997566, + 3.836926271997072, + 3.915554078997957, + 4.008357723996596, + 4.067479856999853, + 4.128980988996773, + 4.192423319997033, + 4.246235474998684, + 4.301449459999276, + 4.3627791209983116, + 4.425496228999691, + 4.487905732999934, + 4.564729760997579, + 4.628986914998677, + 4.698359536996577, + 4.789603017998161, + 4.878943064999476, + 4.9802080409972405, + 5.075376105996838, + 5.166856313997414, + 5.234641233997536, + 5.321722453998518, + 5.4110153679976065, + 5.4871868129994255, + 5.542350603998784, + 5.610478301998228, + 5.696128804996988, + 5.773844041999837, + 5.840940944999602, + 5.907454934997077, + 5.985152329998527, + 6.050242915996932, + 6.118768151998665, + 6.193937526997615, + 6.249646047999704, + 6.309548483997787, + 6.395310789997893, + 6.492602835998696, + 6.591227117998642, + 6.683062737996806, + 6.751811194997572, + 6.831691003997548, + 6.926274135999847, + 7.030602693997935, + 7.115419883997674, + 7.182697440999618, + 7.256560810998053, + 7.326617833998171, + 7.387735339998471, + 7.459630321998702, + 7.523432034999132, + 7.580410612998094, + 7.647706411997206, + 7.716339233997132, + 7.82080152799972 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.74845700076548, + "p95": 98.96143899823073, + "max": 319.3438679991232 + } + }, + { + "start": 30073.484808056, + "first": 30073.570102018, + "last": 30081.30472827, + "end": 30081.304906988, + "elapsed_s": 7.820098931999382, + "chunk_offsets_s": [ + 0.08529396199810435, + 0.4872173510011635, + 0.5732805709994864, + 0.6729265979993215, + 0.7455485700011195, + 0.8035457090009004, + 0.8596220389990776, + 0.925820610998926, + 0.9906463039988012, + 1.075233075000142, + 1.1500927560009586, + 1.2413779320013418, + 1.3254617849997885, + 1.4057583309986512, + 1.4716641849990992, + 1.5399057659997197, + 1.6229828299983637, + 1.710563959000865, + 1.7782583299995167, + 1.860382822000247, + 1.9367337619987666, + 2.004111088997888, + 2.096588010997948, + 2.196307477999653, + 2.2842929449980147, + 2.3603611329999694, + 2.427747167999769, + 2.4895714099984616, + 2.5463568099985423, + 2.6028126509991125, + 2.6684570719990006, + 2.745059755998227, + 2.808002742000099, + 2.8653370949978125, + 2.925056735999533, + 2.9905842099979054, + 3.051230179000413, + 3.1179505369982508, + 3.187598672000604, + 3.25467595200098, + 3.3254493069980526, + 3.383995023999887, + 3.4440715380005713, + 3.517847538998467, + 3.575084464999236, + 3.6395251209978596, + 3.7081869680005184, + 3.769481793999148, + 3.8367189899981895, + 3.914946172000782, + 4.008843069001159, + 4.06702280600075, + 4.12842115300009, + 4.19230371699814, + 4.246066231000441, + 4.300696840000455, + 4.362191393000103, + 4.425149740000052, + 4.487246009000955, + 4.564137263998418, + 4.628883810000843, + 4.6982879739989585, + 4.788653021998471, + 4.87881755700073, + 4.9800231469998835, + 5.075212830997771, + 5.1659345619991655, + 5.234150160998979, + 5.321148183000332, + 5.410490562000632, + 5.48677441000109, + 5.542481123000471, + 5.6107946269985405, + 5.695209374000115, + 5.7730331089987885, + 5.839928778001195, + 5.906720686998597, + 5.984965914998611, + 6.049668613999529, + 6.117910265998944, + 6.193363241000043, + 6.249533309997787, + 6.309558294000453, + 6.3949872169978335, + 6.492241448999266, + 6.59083815899794, + 6.682918490998418, + 6.751353423998808, + 6.830692216000898, + 6.924946348000958, + 7.030270383998868, + 7.115394888998708, + 7.182159304000379, + 7.256047030998161, + 7.326212535001105, + 7.3876413899997715, + 7.45949317999839, + 7.5227472340011445, + 7.580279263998818, + 7.647049407998566, + 7.716243779999786, + 7.819920213998557 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.66184700265876, + "p95": 99.64602699983516, + "max": 401.9233890030591 + } + }, + { + "start": 30073.484621866, + "first": 30073.902440202, + "last": 30081.30540075, + "end": 30081.305464233, + "elapsed_s": 7.820842367000296, + "chunk_offsets_s": [ + 0.41781833600180107, + 0.4874667830008548, + 0.5736684260009497, + 0.6734985790026258, + 0.7459295600019686, + 0.8041529920010362, + 0.8600021000020206, + 0.9264718850026838, + 0.9903901010002301, + 1.0753437970015511, + 1.1504627680005797, + 1.2420338459996856, + 1.3258266290031315, + 1.4061740300021484, + 1.4721119540008658, + 1.5404696820005483, + 1.6231043380030314, + 1.7099197730021842, + 1.7782533690005948, + 1.8608739580013207, + 1.9371902840030089, + 2.0044831020022684, + 2.097034436003014, + 2.1965715210026246, + 2.2843483140022727, + 2.3607714560021122, + 2.42862772800072, + 2.490239183000085, + 2.5469236060016556, + 2.6026441110007, + 2.668928125000093, + 2.7457499810006993, + 2.8088071450001735, + 2.866064334000839, + 2.9251240430021426, + 2.991029275002802, + 3.0522041309995984, + 3.118275918001018, + 3.1879237160028424, + 3.2545652290027647, + 3.3258649750023324, + 3.3847459310018166, + 3.4447865990005084, + 3.518171159001213, + 3.575314742000046, + 3.639674248999654, + 3.708253570002853, + 3.770028123002703, + 3.837236434999795, + 3.915695430001506, + 4.008586994001234, + 4.0677586540004995, + 4.12876858600066, + 4.192635627001437, + 4.246358311000222, + 4.301050034002401, + 4.36276630300199, + 4.425233560999914, + 4.4877445380006975, + 4.564284776002751, + 4.628819239002041, + 4.698706346000108, + 4.789038573002472, + 4.87923480900281, + 4.980255008002132, + 5.075476202000573, + 5.166942968000512, + 5.235729220999929, + 5.321801491001679, + 5.4112708660031785, + 5.487243286999728, + 5.542883941001492, + 5.610539288001746, + 5.695875051002076, + 5.773576526000397, + 5.840705018003064, + 5.9078289970020705, + 5.98572887300179, + 6.050246885002707, + 6.118236127000273, + 6.193474939002044, + 6.249621532002493, + 6.3093070030008676, + 6.3955436130017915, + 6.492487857001834, + 6.590668001001177, + 6.6832756210023945, + 6.751686982002866, + 6.8313194069996825, + 6.92538917300044, + 7.031120572002692, + 7.115514540000731, + 7.182660939000925, + 7.256651785999566, + 7.32671484200182, + 7.387921707002533, + 7.459594859999925, + 7.5233124469996255, + 7.580165948002104, + 7.647305193000648, + 7.715939632002119, + 7.820778884000902 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.63443900147104, + "p95": 98.18014399934327, + "max": 105.73139900225215 + } + }, + { + "start": 30073.484441613, + "first": 30073.735735937, + "last": 30081.304812924, + "end": 30081.305116095, + "elapsed_s": 7.8206744820017775, + "chunk_offsets_s": [ + 0.25129432399990037, + 0.48730795699884766, + 0.5740217630009283, + 0.6734939859998121, + 0.7454288059998362, + 0.8042349889983598, + 0.8602224399983243, + 0.9264876790002745, + 0.9906736990014906, + 1.07548607499848, + 1.1508440620018519, + 1.2427644449999207, + 1.3264000499984832, + 1.406543609999062, + 1.4722444229992107, + 1.5407109529987792, + 1.6231995849993837, + 1.710606316999474, + 1.7785063539995463, + 1.861115276999044, + 1.9375824439994176, + 2.0056268250009452, + 2.09737165900151, + 2.197158577000664, + 2.2844091709994245, + 2.360827912001696, + 2.4287764559994685, + 2.49051726000107, + 2.54727524700138, + 2.6028847740017227, + 2.668518664999283, + 2.745814262998465, + 2.80881244199918, + 2.866125656000804, + 2.9251704980015347, + 2.991274803000124, + 3.052130838001176, + 3.1187007230000745, + 3.187917810999352, + 3.254876464001427, + 3.326152333000209, + 3.3848909939988516, + 3.4449280939988967, + 3.5181124129994714, + 3.57568792299935, + 3.639653908998298, + 3.7090522129983583, + 3.7703135129995644, + 3.8372093420002784, + 3.916208970000298, + 4.009681397001259, + 4.067802533001668, + 4.129211666000629, + 4.192853182001272, + 4.246476426000299, + 4.301348577999306, + 4.363056173999212, + 4.425808855001378, + 4.488277857999492, + 4.564359651001723, + 4.629460320000362, + 4.698813146998873, + 4.789593015000719, + 4.879311652999604, + 4.9805442549986765, + 5.0757483419984055, + 5.166378025998711, + 5.235034001998429, + 5.321735494999302, + 5.411881478001305, + 5.487812515999394, + 5.542659261998779, + 5.610991730001842, + 5.695647524000378, + 5.773469114999898, + 5.840417784998863, + 5.907257525999739, + 5.985581791999721, + 6.050199836001411, + 6.118931362001604, + 6.194234566999512, + 6.250099338001746, + 6.310022817000572, + 6.395664431998739, + 6.492484206999507, + 6.590985637001722, + 6.683327389000624, + 6.751974021000933, + 6.831430384001578, + 6.925837310998759, + 7.031220236000081, + 7.115431709000404, + 7.182626387999335, + 7.256355304001772, + 7.3266400920001615, + 7.38796801799981, + 7.459511452998413, + 7.5232798809993255, + 7.58069057800094, + 7.6477443689982465, + 7.71691878600177, + 7.820371311001509 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.17441700352356, + "p95": 99.47222299888381, + "max": 236.0136329989473 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/receipts/flashnext-cuda-sampled-heads/B1.json b/docs/receipts/flashnext-cuda-sampled-heads/B1.json new file mode 100644 index 0000000000..d9a82d19db --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/B1.json @@ -0,0 +1,23728 @@ +{ + "label": "B1", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "code", + "temperature": 1.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.8, + "per_stream_tps": 67.8, + "ttft_s_max": 0.1, + "request": { + "ttft_s": 0.096, + "decode_tps": 67.75, + "prefill_tps": 145.194, + "itl_mean_ms": 14.76, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.888, + "round_gap_p50_ms": 65.174, + "round_gap_p95_ms": 80.392, + "tokens_per_chunk": 4.414 + }, + "pooled": { + "aggregate_prefill_tps": 145.194, + "itl_mean_ms": 14.76, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.888, + "round_gap_p50_ms": 65.174, + "round_gap_p95_ms": 80.392, + "tokens_per_chunk": 4.414 + }, + "timing_valid": true, + "end_to_end_tps": 66.2803074488693, + "makespan_s": 3.8623840150030446, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.75751534861773, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8599400279999827, + "end_to_end_tps": 66.32227395839766, + "timing_valid": true, + "per_stream_tps": [ + 67.8 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09642279499894357, + "decode_tps": 67.75751534861773, + "prefill_tps": 145.19388283811298, + "itl_mean_ms": 14.758510474519639, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.88828699995065, + "round_gap_p50_ms": 65.17365599938785, + "round_gap_p95_ms": 82.2207730016089, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.758510474519639, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.88828699995065, + "round_gap_p50_ms": 65.17365599938785, + "round_gap_p95_ms": 82.2207730016089, + "round_gap_mean_ms": 66.02491528074576, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 145.19388283811298 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 29021.807791461, + "first": 29021.904214256, + "last": 29025.667634427, + "end": 29025.667731489, + "elapsed_s": 3.8599400279999827, + "chunk_offsets_s": [ + 0.09642279499894357, + 0.1501572679990204, + 0.20449460500094574, + 0.26275851399987005, + 0.3280444859992713, + 0.38884668500031694, + 0.44982369300123537, + 0.5177818419979303, + 0.5900462819990935, + 0.666810071998043, + 0.7313164799998049, + 0.789629345999856, + 0.8453016960011155, + 0.9007509820003179, + 0.9738756379992992, + 1.0491901519999374, + 1.1191639220014622, + 1.199900201998389, + 1.2785847129998729, + 1.348505883001053, + 1.424688357001287, + 1.4931035500012513, + 1.570991837001202, + 1.6497947759999079, + 1.7341539279987046, + 1.802955725001084, + 1.857925778000208, + 1.9117382939984964, + 1.9593137910014775, + 2.0023869369979366, + 2.056017431001237, + 2.1109656660009932, + 2.1710887890003505, + 2.246499663000577, + 2.3116463020014635, + 2.3559281720008585, + 2.402900975997909, + 2.454989903999376, + 2.5036637150005845, + 2.562411767001322, + 2.642089703000238, + 2.728004618998966, + 2.793178274998354, + 2.8430686219981, + 2.9062382910015003, + 2.983746935999079, + 3.065967709000688, + 3.1446606919998885, + 3.221473256999161, + 3.2864594550010224, + 3.3446232249989407, + 3.4069819249998545, + 3.466326264999225, + 3.544416844000807, + 3.622712248001335, + 3.7046739409997826, + 3.784164756998507, + 3.8598429660014517 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 65.17365599938785, + "p95": 82.2207730016089, + "max": 85.91491599872825 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.69677148381446, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8623840150030446, + "end_to_end_tps": 66.2803074488693, + "timing_valid": true, + "per_stream_tps": [ + 67.7 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09551174700027332, + "decode_tps": 67.69677148381446, + "prefill_tps": 146.57882867496852, + "itl_mean_ms": 14.771753188246043, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.46471900100005, + "round_gap_p50_ms": 65.41059099981794, + "round_gap_p95_ms": 79.84439600113546, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.771753188246043, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.46471900100005, + "round_gap_p50_ms": 65.41059099981794, + "round_gap_p95_ms": 79.84439600113546, + "round_gap_mean_ms": 66.08415900004809, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 146.57882867496852 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 29025.668356394, + "first": 29025.763868141, + "last": 29029.530665204, + "end": 29029.530740409, + "elapsed_s": 3.8623840150030446, + "chunk_offsets_s": [ + 0.09551174700027332, + 0.15119164200223167, + 0.2043723080023483, + 0.26283371099998476, + 0.3282443019998027, + 0.38907191100224736, + 0.45128445199952694, + 0.5197533179998572, + 0.5920984120020876, + 0.6672494490012468, + 0.7280865190004988, + 0.7842865920028999, + 0.8440112290008983, + 0.9024368870013859, + 0.9746324080006161, + 1.050004364002234, + 1.121072671001457, + 1.2018926460004877, + 1.2781462550010474, + 1.3493397040001582, + 1.424458138000773, + 1.4922757940003066, + 1.5699691840018204, + 1.6474339030028204, + 1.7272763120017771, + 1.7967882299999474, + 1.8541739020001842, + 1.9111846080013493, + 1.9588661930029048, + 2.0040413179995085, + 2.0575579130018014, + 2.1124699790016166, + 2.1715957630003686, + 2.2493717100005597, + 2.3155580570019083, + 2.3594375920001767, + 2.4070116210023116, + 2.460363096000947, + 2.509361207001348, + 2.566675916001259, + 2.6458304290026717, + 2.7251529790009954, + 2.7894601890002377, + 2.8453950240000268, + 2.9106057150020206, + 2.988067869002407, + 3.069493513001362, + 3.1493379090024973, + 3.2269040110004426, + 3.291837395001494, + 3.351269422000769, + 3.4139538660019753, + 3.472975149001286, + 3.551734861001023, + 3.630737756000599, + 3.7069033450025017, + 3.7834216250012105, + 3.8623088100030145 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 65.41059099981794, + "p95": 79.84439600113546, + "max": 81.42564399895491 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.75032619084826, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.868286798999179, + "end_to_end_tps": 66.17916749767197, + "timing_valid": true, + "per_stream_tps": [ + 67.8 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.1043975880020298, + "decode_tps": 67.75032619084826, + "prefill_tps": 134.102715090772, + "itl_mean_ms": 14.760076537241535, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.89541799866129, + "round_gap_p50_ms": 64.21505600155797, + "round_gap_p95_ms": 80.39164599904325, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.760076537241535, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.89541799866129, + "round_gap_p50_ms": 64.21505600155797, + "round_gap_p95_ms": 80.39164599904325, + "round_gap_mean_ms": 66.03192135081738, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 134.102715090772 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 29029.53126801, + "first": 29029.635665598, + "last": 29033.399485115, + "end": 29033.399554809, + "elapsed_s": 3.868286798999179, + "chunk_offsets_s": [ + 0.1043975880020298, + 0.16050301300128922, + 0.21447831399927964, + 0.27316050700028427, + 0.33859119300177554, + 0.3981692870002007, + 0.46002337300160434, + 0.5268346090015257, + 0.59959944000002, + 0.674201374000404, + 0.7348848990004626, + 0.7905841850006254, + 0.8461295070010237, + 0.9030066080013057, + 0.9803868759991019, + 1.0589756810004474, + 1.130320741001924, + 1.2107123870009673, + 1.2886078049996286, + 1.3600045220009633, + 1.438000681999256, + 1.5063533149987052, + 1.585709508999571, + 1.6645140219989116, + 1.7444028099998832, + 1.8123544709997077, + 1.8666008549989783, + 1.9206280250000418, + 1.9739849360012158, + 2.0205965759996616, + 2.0740667860009125, + 2.1287017989998276, + 2.1887745290005114, + 2.265795582999999, + 2.330010639001557, + 2.3744680080017133, + 2.422805475001951, + 2.4749135470010515, + 2.523800097998901, + 2.582090795000113, + 2.6615062189994205, + 2.740915151000081, + 2.803013249002106, + 2.853627102002065, + 2.9167457750008907, + 3.0009457289997954, + 3.085027755001647, + 3.1639934580016416, + 3.2403682240001217, + 3.3044321950001176, + 3.365452907000872, + 3.4275414399999136, + 3.48748946699925, + 3.564939333999064, + 3.643562239998573, + 3.7198538400007237, + 3.7968764990000636, + 3.868217104998621 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 64.21505600155797, + "p95": 80.39164599904325, + "max": 84.19995399890468 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.7, + "per_stream_tps": 47.6, + "ttft_s_max": 0.2, + "request": { + "ttft_s": 0.143, + "decode_tps": 47.58, + "prefill_tps": 108.076, + "itl_mean_ms": 21.303, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.036, + "round_gap_p50_ms": 80.315, + "round_gap_p95_ms": 100.016, + "tokens_per_chunk": 3.73 + }, + "pooled": { + "aggregate_prefill_tps": 148.705, + "itl_mean_ms": 21.319, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.021, + "round_gap_p50_ms": 79.544, + "round_gap_p95_ms": 101.057, + "tokens_per_chunk": 3.631 + }, + "timing_valid": true, + "end_to_end_tps": 81.79511757019273, + "makespan_s": 6.259542320000037, + "steady_tps": 92.3, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.7856555629242, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.259542320000037, + "end_to_end_tps": 81.79511757019273, + "timing_valid": true, + "steady_tps": 93.38195088990774, + "per_stream_tps": [ + 53.2, + 42.1 + ], + "ttft_s": [ + 0.1, + 0.2 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09886159999950905, + "decode_tps": 53.23938148364982, + "prefill_tps": 141.6121122869701, + "itl_mean_ms": 18.7830882352964, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.37126100197202, + "round_gap_p50_ms": 82.67054300085874, + "round_gap_p95_ms": 99.52149700256996, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19743498000025284, + "decode_tps": 42.06511154790694, + "prefill_tps": 70.9094204075796, + "itl_mean_ms": 23.772669635289667, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.05009800259722, + "round_gap_p50_ms": 76.77648800017778, + "round_gap_p95_ms": 95.84085199821857, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 21.277878935293035, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.05164599895943, + "round_gap_p50_ms": 79.54382900061319, + "round_gap_p95_ms": 99.1534060012782, + "round_gap_mean_ms": 78.06991551798164, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 141.8188408151592 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 29033.401328294, + "first": 29033.500189894, + "last": 29038.289877394, + "end": 29038.290144933, + "elapsed_s": 4.888816638998833, + "chunk_offsets_s": [ + 0.09886159999950905, + 0.2570085680017655, + 0.3133175660004781, + 0.37361996400068165, + 0.4498913870011165, + 0.5208069750005961, + 0.5894432360000792, + 0.6721137790009379, + 0.7559836469990842, + 0.8451012400000764, + 0.9292723910002678, + 0.9903745480005455, + 1.0699183770011587, + 1.1453918059996795, + 1.223332935001963, + 1.3085987660015235, + 1.3951083600004495, + 1.4847129590016266, + 1.5682122940015688, + 1.6482072700018762, + 1.735665094001888, + 1.8144894659999409, + 1.9052999980012828, + 1.9921271150014945, + 2.085008148002089, + 2.1777136890013935, + 2.2438327290001325, + 2.3167826829994738, + 2.3839441939999233, + 2.4504536310014373, + 2.524879691998649, + 2.5965523529994243, + 2.6724052729987307, + 2.7647765340007027, + 2.8459661950000736, + 2.9029567220022727, + 2.958912693000457, + 3.0289794509990315, + 3.1172863470019365, + 3.208337993000896, + 3.2856388540021726, + 3.3660133160010446, + 3.4617169740013196, + 3.548265984001773, + 3.642906930999743, + 3.7418812659998366, + 3.8412880639989453, + 3.9276479510008357, + 4.013589229001809, + 4.102723142001196, + 4.167429836001247, + 4.239849202000187, + 4.320528973999899, + 4.403199290001794, + 4.4984995250015345, + 4.593723456000589, + 4.695448303999001, + 4.794969801001571, + 4.888549100000091 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 83.08493900040048, + "p95": 99.52149700256996, + "max": 158.14696800225647 + } + }, + { + "start": 29033.401211702, + "first": 29033.598646682, + "last": 29039.660677439, + "end": 29039.660754022, + "elapsed_s": 6.259542320000037, + "chunk_offsets_s": [ + 0.19743498000025284, + 0.25687415699940175, + 0.3135236800007988, + 0.3734800339989306, + 0.45008599599896115, + 0.5211579060014628, + 0.5901203700013866, + 0.6726100539999607, + 0.7563404649990844, + 0.8449206490004144, + 0.9296731790018384, + 0.9907115659989358, + 1.07009811700118, + 1.1458316659991397, + 1.2235269049997441, + 1.3086350110024796, + 1.3950053070002468, + 1.4851014060004673, + 1.5681320440016862, + 1.6481031160001294, + 1.7355015339999227, + 1.8148634400022274, + 1.905652989000373, + 1.9925215460025356, + 2.085449428999709, + 2.177582552001695, + 2.243584103001922, + 2.316699017999781, + 2.3838155939993158, + 2.450663136001822, + 2.5248185810014547, + 2.596469154999795, + 2.6722409870017145, + 2.7645916610017593, + 2.8460362190016895, + 2.9030124460005027, + 2.958982907999598, + 3.0290373940006248, + 3.1173272619998897, + 3.208377360002487, + 3.286084008999751, + 3.366323905000172, + 3.4617425889991864, + 3.548280106002494, + 3.642940270001418, + 3.7419362799992086, + 3.8415909690011176, + 3.927964582999266, + 4.013628138000058, + 4.102760886999022, + 4.16759129500133, + 4.239600652999798, + 4.320403452999017, + 4.403235844001756, + 4.498375539002154, + 4.594216391000373, + 4.695871625001018, + 4.795025031002297, + 4.88818523900045, + 4.945087169999169, + 4.994899343000725, + 5.03988967999976, + 5.099454025999876, + 5.163510670001415, + 5.222077362999698, + 5.280931150002289, + 5.358223277999059, + 5.417981483999029, + 5.4649317910007085, + 5.510601769001369, + 5.557102714999928, + 5.616286049000337, + 5.694879360002233, + 5.7716558480024105, + 5.8438605980009015, + 5.917761349999637, + 5.991556926001067, + 6.0557509810023475, + 6.105340856000112, + 6.156031326001539, + 6.202665418000834, + 6.259465736999118 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 76.77648800017778, + "p95": 95.84085199821857, + "max": 101.65523400064558 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.54498849972015, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.273602264998772, + "end_to_end_tps": 81.61180425104622, + "timing_valid": true, + "steady_tps": 92.06354089834974, + "per_stream_tps": [ + 54.0, + 41.3 + ], + "ttft_s": [ + 0.19, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18781949299955158, + "decode_tps": 54.01160709052823, + "prefill_tps": 74.5396538794481, + "itl_mean_ms": 18.514538890203944, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.02139600145165, + "round_gap_p50_ms": 82.85028699901886, + "round_gap_p95_ms": 102.7410120004788, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09477353500187746, + "decode_tps": 41.272494249860074, + "prefill_tps": 147.7205635489133, + "itl_mean_ms": 24.22921168626464, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.84736900008284, + "round_gap_p50_ms": 77.95965100012836, + "round_gap_p95_ms": 99.60851200230536, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 21.371875288234293, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.02139600145165, + "round_gap_p50_ms": 79.27748400106793, + "round_gap_p95_ms": 102.08177299864474, + "round_gap_mean_ms": 78.4147942230179, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 148.96418419559177 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 29039.661854315, + "first": 29039.849673808, + "last": 29044.570881225, + "end": 29044.571060849, + "elapsed_s": 4.909206534000987, + "chunk_offsets_s": [ + 0.18781949299955158, + 0.24589156000001822, + 0.30333761400106596, + 0.3634946470010618, + 0.4413646540015179, + 0.5103737919998821, + 0.5795228440001665, + 0.6623731309991854, + 0.745744644998922, + 0.8411245950010198, + 0.9276786269983859, + 0.9899633540007926, + 1.0691745930016623, + 1.1392191099985212, + 1.2182333439996, + 1.3046616979991086, + 1.391351594000298, + 1.4773944160006067, + 1.5627992330009874, + 1.6423343300011766, + 1.7284837219995097, + 1.809066261001135, + 1.905666853999719, + 1.9930442869990657, + 2.0840656830005173, + 2.172016463999171, + 2.239241535000474, + 2.311988501001906, + 2.378594368001359, + 2.447426999999152, + 2.521985969000525, + 2.594196597001428, + 2.669726897998771, + 2.7623214529994584, + 2.8502401010009635, + 2.911584249999578, + 2.970438568001555, + 3.039618809998501, + 3.1223280720005278, + 3.211421572999825, + 3.2891275549991406, + 3.3711800050004967, + 3.4664840170007665, + 3.5523608760013303, + 3.6466811889986275, + 3.7453609699987283, + 3.849664640998526, + 3.940175222000107, + 4.027680448998581, + 4.112721260000399, + 4.174959383999521, + 4.249016485999164, + 4.32823594400179, + 4.411675567000202, + 4.507112851999409, + 4.60232654200081, + 4.705067554001289, + 4.804273964000458, + 4.9090269100015576 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 83.11090049937775, + "p95": 102.7410120004788, + "max": 104.75294600109919 + } + }, + { + "start": 29039.661709163, + "first": 29039.756482698, + "last": 29045.934931678, + "end": 29045.935311428, + "elapsed_s": 6.273602264998772, + "chunk_offsets_s": [ + 0.09477353500187746, + 0.24639757100158022, + 0.3033187580003869, + 0.3638345110011869, + 0.44132960899878526, + 0.5101841110008536, + 0.5793482670014782, + 0.6623554889993102, + 0.7461025980010163, + 0.8410572329994466, + 0.9277366179994715, + 0.9898732239998935, + 1.0691507080009615, + 1.1391540900003747, + 1.2181944320000184, + 1.3045461399997293, + 1.3911273570010962, + 1.4777718879995518, + 1.5627536940010032, + 1.6422799610008951, + 1.7283779680001317, + 1.8089088660017296, + 1.9056832079986634, + 1.9931230440015497, + 2.083723011000984, + 2.172418007001397, + 2.23908655500054, + 2.3118358040010207, + 2.3788248609998845, + 2.4477429789985763, + 2.522460317999503, + 2.594570847999421, + 2.6701388059991586, + 2.7629861749992415, + 2.850618360000226, + 2.9118992069998058, + 2.970694708001247, + 3.0395617229987693, + 3.1225728419994994, + 3.2113291730020137, + 3.2890793939986906, + 3.3711589310005365, + 3.4671931359989685, + 3.5528109619990573, + 3.646571875000518, + 3.7457762070007448, + 3.8495918850021553, + 3.9405495099999825, + 4.028069940999558, + 4.113015899998572, + 4.175448180001695, + 4.249428007999086, + 4.3279291600010765, + 4.411992564000684, + 4.5069419290011865, + 4.602894560001005, + 4.7049763329996495, + 4.804584845001955, + 4.909832659999665, + 4.966501261002122, + 5.01621096099916, + 5.060884982998687, + 5.114990303998638, + 5.176731660001678, + 5.2356269330011855, + 5.294525672001328, + 5.371184055999038, + 5.431651267001143, + 5.476380120999238, + 5.521738539999205, + 5.567934769998828, + 5.629417625001224, + 5.706208269999479, + 5.783192769999005, + 5.861152420999133, + 5.939181349000137, + 6.017473504998634, + 6.081915565999225, + 6.127066138000373, + 6.173899663001066, + 6.220968303001428, + 6.273222514999361 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 77.95965100012836, + "p95": 99.60851200230536, + "max": 151.62403599970276 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.74628882732185, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.258341190001374, + "end_to_end_tps": 81.8108160702385, + "timing_valid": true, + "steady_tps": 92.28214617380291, + "per_stream_tps": [ + 53.1, + 42.0 + ], + "ttft_s": [ + 0.09, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09453608900003019, + "decode_tps": 53.094854832075576, + "prefill_tps": 148.09159283070753, + "itl_mean_ms": 18.834216670574296, + "itl_p50_ms": 0.0, + "itl_p95_ms": 90.64405399840325, + "round_gap_p50_ms": 83.82029999847873, + "round_gap_p95_ms": 101.78092200294486, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18829283699960797, + "decode_tps": 42.010022945020985, + "prefill_tps": 74.35227076656744, + "itl_mean_ms": 23.803843223525774, + "itl_p50_ms": 0.0, + "itl_p95_ms": 89.93449300032808, + "round_gap_p50_ms": 77.66554000045289, + "round_gap_p95_ms": 100.42401800092193, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 21.319029947050034, + "itl_p50_ms": 0.0, + "itl_p95_ms": 89.93449300032808, + "round_gap_p50_ms": 79.98481300091953, + "round_gap_p95_ms": 101.05673600264709, + "round_gap_mean_ms": 78.22090124457208, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 148.7045415331349 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 29045.936606718, + "first": 29046.031142807, + "last": 29050.833868058, + "end": 29050.834028667, + "elapsed_s": 4.897421948997362, + "chunk_offsets_s": [ + 0.09453608900003019, + 0.24753933899773983, + 0.3034713839988399, + 0.3620077789964853, + 0.4395156649989076, + 0.5096519099970465, + 0.5828558829998656, + 0.6687815749974106, + 0.7535226009968028, + 0.842492202998983, + 0.9263125029974617, + 0.9871788729979016, + 1.0652582419970713, + 1.1340823599966825, + 1.2108579789965006, + 1.2985400069992465, + 1.383613082998636, + 1.469823056999303, + 1.561200309999549, + 1.6440810799977044, + 1.733148445997358, + 1.8115201229993545, + 1.9032942519988865, + 1.9902152229988133, + 2.080052801997226, + 2.16831198699947, + 2.2326734049966035, + 2.303869956998824, + 2.37025350699696, + 2.4381837529981567, + 2.511931699998968, + 2.5883681219966093, + 2.668720929999836, + 2.759364983998239, + 2.84254250199956, + 2.901187799998297, + 2.9592339749979146, + 3.0278875839976536, + 3.110788848000084, + 3.1985296699967876, + 3.276271998998709, + 3.3590825739993306, + 3.4548074799968163, + 3.543360623996705, + 3.643961882997246, + 3.745018618999893, + 3.8458496219973313, + 3.933601048996934, + 4.019823783997708, + 4.104087093997805, + 4.166238268997404, + 4.239802935997432, + 4.319728019996546, + 4.404235242996947, + 4.498342338996736, + 4.6001232609996805, + 4.705579694997141, + 4.803985341997759, + 4.897261339996476 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 84.0418049992877, + "p95": 101.78092200294486, + "max": 153.00324999770964 + } + }, + { + "start": 29045.936288359, + "first": 29046.124581196, + "last": 29052.194561218, + "end": 29052.194629549, + "elapsed_s": 6.258341190001374, + "chunk_offsets_s": [ + 0.18829283699960797, + 0.24805249400014873, + 0.30356873300115694, + 0.362543570001435, + 0.43940906599891605, + 0.5102768869983265, + 0.5832673580007395, + 0.6690252359985607, + 0.7536222189992259, + 0.842536469001061, + 0.9264704980014358, + 0.987235837001208, + 1.065277118999802, + 1.1342829130007885, + 1.2110900209991087, + 1.2984908600010385, + 1.383669981998537, + 1.4698556729999837, + 1.5611326779981027, + 1.6446201920007297, + 1.7329943859986088, + 1.8119710329992813, + 1.9032840649997524, + 1.9908145660010632, + 2.0807490590013913, + 2.1689402320007503, + 2.233218442001089, + 2.3040668290013855, + 2.3709181470003386, + 2.4387287579993426, + 2.5120808009996836, + 2.588595492998138, + 2.6692168549998314, + 2.759638448998885, + 2.8431150670003262, + 2.901766740000312, + 2.959511046999978, + 3.0279286810000485, + 3.1112986839980294, + 3.1989956309989793, + 3.276661170999432, + 3.3595212449981773, + 3.4546966449997853, + 3.5432446929989965, + 3.643988633000845, + 3.745516248000058, + 3.84594026600098, + 3.9341245089999575, + 4.020413172998815, + 4.104675955000857, + 4.166746229999262, + 4.240325277998636, + 4.3201652280004055, + 4.404153465999116, + 4.499082788999658, + 4.6002235429987195, + 4.706244631001027, + 4.804412180001236, + 4.897706527000992, + 4.954697930999828, + 5.0033860270013975, + 5.048199314998783, + 5.102122806998523, + 5.162874735000514, + 5.220856656000251, + 5.2791498100014, + 5.357553040001221, + 5.4180660269994405, + 5.463121818000218, + 5.508853923998686, + 5.5586448419999215, + 5.622965844999271, + 5.701709404998837, + 5.781694217999757, + 5.853743815998314, + 5.929020717001549, + 6.005164098998648, + 6.068367963998753, + 6.112727964999067, + 6.159365310999419, + 6.206209727999521, + 6.25827285899868 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 77.66554000045289, + "p95": 100.42401800092193, + "max": 106.0210880023078 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 119.2, + "per_stream_tps": 33.8, + "ttft_s_max": 0.4, + "request": { + "ttft_s": 0.237, + "decode_tps": 33.742, + "prefill_tps": 60.95, + "itl_mean_ms": 29.659, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.788, + "round_gap_p50_ms": 108.45, + "round_gap_p95_ms": 139.361, + "tokens_per_chunk": 3.633 + }, + "pooled": { + "aggregate_prefill_tps": 146.053, + "itl_mean_ms": 29.668, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.085, + "round_gap_p50_ms": 108.805, + "round_gap_p95_ms": 138.445, + "tokens_per_chunk": 3.618 + }, + "timing_valid": true, + "end_to_end_tps": 118.30378548183461, + "makespan_s": 8.655682451997563, + "steady_tps": 134.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 118.97896314511074, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 8.668534820997593, + "end_to_end_tps": 118.12838284038362, + "timing_valid": true, + "steady_tps": 134.42629981747322, + "per_stream_tps": [ + 38.1, + 30.4, + 34.7, + 32.4 + ], + "ttft_s": [ + 0.19, + 0.28, + 0.4, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18887878800160252, + "decode_tps": 38.130408632858924, + "prefill_tps": 74.12161073312912, + "itl_mean_ms": 26.225787654901996, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.81379800030845, + "round_gap_p50_ms": 111.98413700185483, + "round_gap_p95_ms": 140.11640700118733, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28317314500236535, + "decode_tps": 30.413265386259035, + "prefill_tps": 49.43971646705078, + "itl_mean_ms": 32.88038910980629, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.05294799848343, + "round_gap_p50_ms": 106.12782200041693, + "round_gap_p95_ms": 137.7020200015977, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.40092832799928146, + "decode_tps": 34.71078208913707, + "prefill_tps": 34.91895938075269, + "itl_mean_ms": 28.809492031381097, + "itl_p50_ms": 0.0, + "itl_p95_ms": 124.76545300160069, + "round_gap_p50_ms": 108.09495600187802, + "round_gap_p95_ms": 137.4298840019037, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09529690999988816, + "decode_tps": 32.35240047345459, + "prefill_tps": 146.9092754425766, + "itl_mean_ms": 30.909607490192524, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.47078900024644, + "round_gap_p50_ms": 108.8052470004186, + "round_gap_p95_ms": 138.60482500240323, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 29.706319071570476, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.01659900226514, + "round_gap_p50_ms": 108.8052470004186, + "round_gap_p95_ms": 138.44523599982494, + "round_gap_mean_ms": 108.60374714337593, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 139.38270015615075 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 29052.196250297, + "first": 29052.385129085, + "last": 29059.072704937, + "end": 29059.073111312, + "elapsed_s": 6.876861015000031, + "chunk_offsets_s": [ + 0.18887878800160252, + 0.4684679340025468, + 0.537336489000154, + 0.6123290560026362, + 0.6994725110016589, + 0.7788278800035187, + 0.8670897310003056, + 0.9733414750007796, + 1.0767434120025428, + 1.184596656003123, + 1.2930526550007926, + 1.3972101560029841, + 1.50280041500082, + 1.5905830130031973, + 1.688122375002422, + 1.7991007230011746, + 1.922519611001917, + 2.044736965002812, + 2.1628482320011244, + 2.2748323690029792, + 2.385567412002274, + 2.495193136001035, + 2.6202097350033, + 2.7503604220000852, + 2.883237148002081, + 3.0233535550032684, + 3.14764129300238, + 3.267103029000282, + 3.379788156002178, + 3.4680759540024155, + 3.5706641460019455, + 3.6748711520012876, + 3.787900011000602, + 3.9137138090009103, + 4.034507727003074, + 4.141212754002481, + 4.246207050000521, + 4.358431777000078, + 4.476634880000347, + 4.600636507002491, + 4.7106923470018955, + 4.834648892003315, + 4.963734105000185, + 5.085156022003503, + 5.197013232002064, + 5.324894729001244, + 5.455067842001881, + 5.573737403003179, + 5.688281074002589, + 5.794369496001309, + 5.876854326001194, + 5.982858528001088, + 6.090468995000265, + 6.191734401003487, + 6.318656130002637, + 6.455130720001762, + 6.600128215002769, + 6.738573451002594, + 6.8764546400016116 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 112.10443200070586, + "p95": 140.11640700118733, + "max": 279.5891460009443 + } + }, + { + "start": 29052.196705606, + "first": 29052.479878751, + "last": 29060.864377974, + "end": 29060.864671786, + "elapsed_s": 8.667966180000803, + "chunk_offsets_s": [ + 0.28317314500236535, + 0.46860320700216107, + 0.5369550410032389, + 0.6113263190018188, + 0.6992339680000441, + 0.7782712190019083, + 0.867198151001503, + 0.9722545980002906, + 1.0763562450010795, + 1.1843839540015324, + 1.292013592999865, + 1.3969444610011124, + 1.502008339000895, + 1.589842299999873, + 1.6876054090025718, + 1.7985870010015788, + 1.9223459620006906, + 2.0445102420017065, + 2.162130504002562, + 2.2746229540025524, + 2.3843656000026385, + 2.494663560002664, + 2.619493243000761, + 2.7497219720025896, + 2.882854526000301, + 3.0229372909998347, + 3.146824639999977, + 3.266387161002058, + 3.3792150730005233, + 3.4674353310001607, + 3.5702761390020896, + 3.6746485700023186, + 3.7877895490019, + 3.9128424970003834, + 4.0339457640002365, + 4.140630637000868, + 4.245816009002738, + 4.35787831700145, + 4.476131814000837, + 4.600078647999908, + 4.710365593000461, + 4.8334387310023885, + 4.963359225002932, + 5.084745403000852, + 5.196734876000846, + 5.324644904001616, + 5.454445441002463, + 5.573204366002756, + 5.6880213460026425, + 5.7941491680030595, + 5.876478873000451, + 5.982612881001842, + 6.090395225000975, + 6.191148582001915, + 6.318269808001787, + 6.455161885001871, + 6.600226290000137, + 6.738171177999902, + 6.8758731980015, + 6.965046285000426, + 7.040161921002436, + 7.11575730000186, + 7.200210545001028, + 7.300242327000888, + 7.406412434000231, + 7.487243045001378, + 7.5811354640027275, + 7.668039393000072, + 7.747835025002132, + 7.817238342002383, + 7.888481545000104, + 7.9773382150015095, + 8.04220339600215, + 8.119247116999759, + 8.186638500003028, + 8.253433147001488, + 8.330987493001885, + 8.410204165000323, + 8.475068178002402, + 8.519667041000503, + 8.565846674002387, + 8.61367151200102, + 8.667672368002968 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 106.13091500090377, + "p95": 137.7020200015977, + "max": 185.43006199979573 + } + }, + { + "start": 29052.196980162, + "first": 29052.59790849, + "last": 29059.944328958, + "end": 29059.944806797, + "elapsed_s": 7.74782663500082, + "chunk_offsets_s": [ + 0.40092832799928146, + 0.4681796510012646, + 0.5370276229987212, + 0.6108488840000064, + 0.6985534949999419, + 0.7782154589986021, + 0.8664840870005719, + 0.9718128650019935, + 1.076159820000612, + 1.1837629149995337, + 1.2918578710014117, + 1.396344324999518, + 1.5018148060007661, + 1.5897590200002014, + 1.6878399170018383, + 1.798884061001445, + 1.922361697001179, + 2.044690316000924, + 2.161374325001816, + 2.273837543998525, + 2.3846598630007065, + 2.4948800089987344, + 2.619645462000335, + 2.749050399001135, + 2.8827272950002225, + 3.0228120729989314, + 3.14712440799849, + 3.2665229980011645, + 3.378675817999465, + 3.4675832989996707, + 3.570251034998364, + 3.674302379000437, + 3.787383774000773, + 3.91296391799915, + 4.033327462999296, + 4.14085442799842, + 4.246307146000618, + 4.357473724001466, + 4.476183868999215, + 4.600229236999439, + 4.710204568000336, + 4.833379697000055, + 4.9633125770014885, + 5.084774661001575, + 5.1961166270011745, + 5.323927429999458, + 5.454047827999602, + 5.573569469001086, + 5.6882888239997556, + 5.794149441000627, + 5.876585727000929, + 5.9829592400019465, + 6.089565958998719, + 6.19078330100092, + 6.3182663890002, + 6.454608664000261, + 6.599651784999878, + 6.738494985998841, + 6.875924870000745, + 6.964437476999592, + 7.040140463999705, + 7.115796008998586, + 7.199721332999616, + 7.3001749940012814, + 7.405907700998796, + 7.486685995001608, + 7.580528302001767, + 7.66794618700078, + 7.747348796001461 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 109.03514350138721, + "p95": 137.4298840019037, + "max": 145.0431209996168 + } + }, + { + "start": 29052.196136965, + "first": 29052.291433875, + "last": 29060.173383785, + "end": 29060.173518635, + "elapsed_s": 7.977381669999886, + "chunk_offsets_s": [ + 0.09529690999988816, + 0.46897353299937095, + 0.5376362139977573, + 0.6114919669998926, + 0.6992048249994696, + 0.7790109529996698, + 0.8669670399976894, + 0.9733787689983728, + 1.0770806369982893, + 1.1843093289971875, + 1.2931145759976062, + 1.3970985879968794, + 1.5023540289985249, + 1.5909934849987621, + 1.6883846719974827, + 1.7994094459972985, + 1.9231482229988615, + 2.045323306996579, + 2.162499277997995, + 2.2745970929972827, + 2.385240592997434, + 2.4953810699989845, + 2.62024628799918, + 2.7495500249970064, + 2.883497772996634, + 3.0237829689976934, + 3.1470358529986697, + 3.2675554579982418, + 3.3796088359995338, + 3.4682337649974215, + 3.5710064089980733, + 3.6753785269975197, + 3.7882712749997154, + 3.913742063999962, + 4.03408663799928, + 4.141493778999575, + 4.246710380997683, + 4.358230709000054, + 4.4773325309979555, + 4.6004535849970125, + 4.711444602999109, + 4.834448292996967, + 4.964220347999799, + 5.085450590999244, + 5.196846644998004, + 5.324884971996653, + 5.455088069997146, + 5.574575489998097, + 5.689474637998501, + 5.795157640997786, + 5.877675125997484, + 5.9835258969978895, + 6.09070278299987, + 6.1917960249993484, + 6.318984941997769, + 6.455515876998106, + 6.60058984999705, + 6.739194674999453, + 6.87692177399731, + 6.965212483999494, + 7.041053352997551, + 7.11612156299816, + 7.200663521998649, + 7.300909240999317, + 7.406530934997136, + 7.487581908997527, + 7.581106954999996, + 7.668373470998631, + 7.7479797919986595, + 7.817563994998636, + 7.888844203996996, + 7.977246819998982 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 108.8052470004186, + "p95": 138.60482500240323, + "max": 373.6766229994828 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 119.25159126727308, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 8.648685630996624, + "end_to_end_tps": 118.39949371382113, + "timing_valid": true, + "steady_tps": 135.0525811321754, + "per_stream_tps": [ + 38.7, + 29.8, + 34.7, + 32.8 + ], + "ttft_s": [ + 0.28, + 0.1, + 0.37, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.27964826799870934, + "decode_tps": 38.7410452747661, + "prefill_tps": 50.062888285310656, + "itl_mean_ms": 25.812416596083636, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.76212199928705, + "round_gap_p50_ms": 111.79876000096556, + "round_gap_p95_ms": 143.19063600123627, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.0951990060020762, + "decode_tps": 29.81289781681827, + "prefill_tps": 147.0603590093648, + "itl_mean_ms": 33.5425293490213, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.63140299971565, + "round_gap_p50_ms": 105.02510699734557, + "round_gap_p95_ms": 140.94208299866295, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.37138884399973904, + "decode_tps": 34.66339455241365, + "prefill_tps": 37.69633963482715, + "itl_mean_ms": 28.848876831376828, + "itl_p50_ms": 0.0, + "itl_p95_ms": 124.93709699992905, + "round_gap_p50_ms": 107.43508400264545, + "round_gap_p95_ms": 140.30843600266962, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18846061099975486, + "decode_tps": 32.819784396998536, + "prefill_tps": 74.28607986428639, + "itl_mean_ms": 30.469426243136837, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.98300299941911, + "round_gap_p50_ms": 107.58723199978704, + "round_gap_p95_ms": 140.72601400039275, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 29.668312254904652, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.08471999899484, + "round_gap_p50_ms": 107.58723199978704, + "round_gap_p95_ms": 140.94208299866295, + "round_gap_mean_ms": 108.46479749104927, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 150.60711651730412 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 29060.865994723, + "first": 29061.145642991, + "last": 29067.727809223, + "end": 29067.728012792, + "elapsed_s": 6.862018068997713, + "chunk_offsets_s": [ + 0.27964826799870934, + 0.43706270099937683, + 0.5068532029981725, + 0.5832075090002036, + 0.6757024859980447, + 0.7586653250000381, + 0.8467470779978612, + 0.9518147079979826, + 1.0580783580007846, + 1.1645961989997886, + 1.2708097169997927, + 1.369355311999243, + 1.4727746680000564, + 1.560552334998647, + 1.6630617929986329, + 1.7748605529995984, + 1.8981716679991223, + 2.0211910769976384, + 2.137848122998548, + 2.2498955269984435, + 2.3530504230002407, + 2.4619424359989353, + 2.5881247889992665, + 2.7243887369986624, + 2.85928185100056, + 3.0002301770000486, + 3.124539645999903, + 3.245124033997854, + 3.349595966999914, + 3.44004498600043, + 3.542119587000343, + 3.651211656997475, + 3.767609607999475, + 3.893371729998762, + 4.016929949997575, + 4.125657336000586, + 4.229927795000549, + 4.3373776669977815, + 4.450957114997436, + 4.575711576999311, + 4.692753454997728, + 4.819455027998629, + 4.951440852997621, + 5.074648535999586, + 5.186286544998438, + 5.3107122629990045, + 5.437485433998518, + 5.551853550998203, + 5.670305670999369, + 5.777465287999803, + 5.860233518000314, + 5.966514735999226, + 6.074478155998804, + 6.175790266999684, + 6.299078791998909, + 6.4321749480004655, + 6.577896842998598, + 6.721087478999834, + 6.861814500000037 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 111.92308200043044, + "p95": 143.19063600123627, + "max": 157.4144330006675 + } + }, + { + "start": 29060.86584625, + "first": 29060.961045256, + "last": 29069.51439024, + "end": 29069.514447542, + "elapsed_s": 8.648601291999512, + "chunk_offsets_s": [ + 0.0951990060020762, + 0.4366986170025484, + 0.5066674040026555, + 0.5837570240000787, + 0.6757465100017725, + 0.7587009400012903, + 0.8475256170022476, + 0.9525507239995932, + 1.0580489250023675, + 1.1644218060027924, + 1.2710258420011087, + 1.3693713749999006, + 1.4725559460021032, + 1.5605857100017602, + 1.6633730659996218, + 1.7753721440021764, + 1.8981137959999614, + 2.0210774240003957, + 2.138278640999488, + 2.250330632999976, + 2.353882818002603, + 2.462531122000655, + 2.588162525000371, + 2.7241731560025073, + 2.859069931000704, + 3.000012013999367, + 3.124536118000833, + 3.2450536350006587, + 3.3495441600025515, + 3.440315276002366, + 3.5424915850016987, + 3.6516031900027883, + 3.7672630890010623, + 3.8938506770027743, + 4.017185136002809, + 4.126136402999691, + 4.229962872002943, + 4.337125698999444, + 4.451419507000537, + 4.57557813600215, + 4.692705192999711, + 4.819903463001538, + 4.950913832999504, + 5.074468631002674, + 5.186364897999738, + 5.310207881000679, + 5.437416705000942, + 5.551833711000654, + 5.670297853001102, + 5.777774433001468, + 5.860247857999639, + 5.9659526590003225, + 6.074349853002786, + 6.175849721999839, + 6.299070891000156, + 6.432023287001357, + 6.5778435050015105, + 6.721374148000905, + 6.8625592600001255, + 6.950195968001935, + 7.02702469200085, + 7.102026680000563, + 7.187076976002572, + 7.285464224001771, + 7.382866302999901, + 7.461575176999759, + 7.553540425000392, + 7.642874943001516, + 7.728663379999489, + 7.797636940002121, + 7.868871897000645, + 7.958547119000286, + 8.023708398002782, + 8.10090947900244, + 8.168551435002882, + 8.234947008000745, + 8.30728608500067, + 8.38295151100101, + 8.445718926999689, + 8.490221457002917, + 8.537171728999965, + 8.584288772002765, + 8.648543990002509 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 105.26165400005993, + "p95": 140.94208299866295, + "max": 341.4996110004722 + } + }, + { + "start": 29060.866201446, + "first": 29061.23759029, + "last": 29068.594053882, + "end": 29068.594125923, + "elapsed_s": 7.72792447699976, + "chunk_offsets_s": [ + 0.37138884399973904, + 0.4367216999999073, + 0.5068691250016855, + 0.5830570070029353, + 0.6757627830011188, + 0.7586517370000365, + 0.8468195730019943, + 0.9518233939998026, + 1.0577586630024598, + 1.1646098240016727, + 1.2704849550027575, + 1.3690906570009247, + 1.4727911869995296, + 1.5603987169997708, + 1.662544897000771, + 1.7747702070009836, + 1.8980208320026577, + 2.0210344870029076, + 2.137486333002016, + 2.249616466000589, + 2.3530711679995875, + 2.462076203999459, + 2.588208200002555, + 2.723900337001396, + 2.8590188560010574, + 2.999925776002783, + 3.124392057001387, + 3.2448572339999373, + 3.3493278240021027, + 3.439775370999996, + 3.541848934000882, + 3.651365623001766, + 3.7673494260016014, + 3.893400824999844, + 4.016656561001582, + 4.125293414002954, + 4.229948964999494, + 4.337384049002139, + 4.45068762700248, + 4.5756247240024095, + 4.692768390003039, + 4.8193636819996755, + 4.951001484001608, + 5.074239413999749, + 5.186275537002075, + 5.3109770030023356, + 5.43698002200108, + 5.551690234002308, + 5.670161267000367, + 5.777167661999556, + 5.859746091002307, + 5.9659581250016345, + 6.074594653000531, + 6.1757196940016, + 6.299089611002273, + 6.4322347060005995, + 6.577566776002641, + 6.721077961999981, + 6.861386398002651, + 6.949977356001909, + 7.0271116540025105, + 7.101900861001923, + 7.187139349000063, + 7.284783451999829, + 7.382998879002116, + 7.461442276002344, + 7.55306514500262, + 7.642125983002188, + 7.7278524360008305 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 108.03580600077112, + "p95": 140.30843600266962, + "max": 145.3320700020413 + } + }, + { + "start": 29060.865761911, + "first": 29061.054222522, + "last": 29068.823926214, + "end": 29068.824066701, + "elapsed_s": 7.958304789997783, + "chunk_offsets_s": [ + 0.18846061099975486, + 0.4372566519996326, + 0.5071867229999043, + 0.5837642019978375, + 0.6759858749974228, + 0.7592339259972505, + 0.8473823709973658, + 0.9528583369974513, + 1.0580560199996398, + 1.1649008989988943, + 1.2709820549971482, + 1.3696323730000586, + 1.4727424509983393, + 1.5608800549998705, + 1.6630711229990993, + 1.775162742997054, + 1.8983413039968582, + 2.021355227996537, + 2.1380049839972344, + 2.2501863529978436, + 2.3533975039972574, + 2.4622340299974894, + 2.588600527997187, + 2.7245676109996566, + 2.860019187999569, + 3.000613517997408, + 3.1247241139972175, + 3.2453997699994943, + 3.349879584999144, + 3.4405944919999456, + 3.5427044559983187, + 3.6515568219983834, + 3.767573381999682, + 3.8941046499967342, + 4.016845101999934, + 4.125670935998642, + 4.229960513999686, + 4.337547745999473, + 4.451066139998147, + 4.576150859997142, + 4.69312656799957, + 4.819875093999144, + 4.951382185998227, + 5.0749389979973785, + 5.186637022998184, + 5.311185777998617, + 5.437168780998036, + 5.552046890999918, + 5.670474070997443, + 5.7775383239968505, + 5.860512004997872, + 5.966529077999439, + 6.074819794997893, + 6.176074878996587, + 6.299593367999478, + 6.432847416999721, + 6.577593847996468, + 6.721390555998369, + 6.862116569998761, + 6.950479242997972, + 7.0273214769986225, + 7.102806967999641, + 7.187351201999263, + 7.285605319997558, + 7.383255353997811, + 7.461942116999126, + 7.5532180499976675, + 7.642723285996908, + 7.7281364089976705, + 7.797838485999819, + 7.868767339998158, + 7.9581643029996485 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 107.58723199978704, + "p95": 140.72601400039275, + "max": 248.79604099987773 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 119.20591440118584, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 8.655682451997563, + "end_to_end_tps": 118.30378548183461, + "timing_valid": true, + "steady_tps": 133.939037654386, + "per_stream_tps": [ + 37.7, + 30.5, + 34.7, + 32.8 + ], + "ttft_s": [ + 0.1, + 0.29, + 0.38, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09887817400158383, + "decode_tps": 37.74027709273349, + "prefill_tps": 141.58837520377094, + "itl_mean_ms": 26.49689077647339, + "itl_p50_ms": 0.0, + "itl_p95_ms": 127.281862001837, + "round_gap_p50_ms": 112.55806299959659, + "round_gap_p95_ms": 144.56247700218228, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28893298500042874, + "decode_tps": 30.480253890765336, + "prefill_tps": 48.45414240253402, + "itl_mean_ms": 32.80812566666225, + "itl_p50_ms": 0.0, + "itl_p95_ms": 127.02759899912053, + "round_gap_p50_ms": 104.67914699984249, + "round_gap_p95_ms": 137.83386000068276, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38314718499896117, + "decode_tps": 34.69502072600017, + "prefill_tps": 36.539482862279044, + "itl_mean_ms": 28.822579698031653, + "itl_p50_ms": 0.0, + "itl_p95_ms": 126.47071099854656, + "round_gap_p50_ms": 108.80711099889595, + "round_gap_p95_ms": 137.6422100001946, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19488668099802453, + "decode_tps": 32.81515555197781, + "prefill_tps": 71.83661771192004, + "itl_mean_ms": 30.473724203928963, + "itl_p50_ms": 0.0, + "itl_p95_ms": 127.02821399943787, + "round_gap_p50_ms": 108.98579700005939, + "round_gap_p95_ms": 138.0534109994187, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 29.650330086274064, + "itl_p50_ms": 0.0, + "itl_p95_ms": 126.73099499806995, + "round_gap_p50_ms": 108.98579700005939, + "round_gap_p95_ms": 137.87312999920687, + "round_gap_mean_ms": 108.39905622938905, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 146.05336120108086 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 29069.516195342, + "first": 29069.615073516, + "last": 29076.371780664, + "end": 29076.372418949, + "elapsed_s": 6.856223606999265, + "chunk_offsets_s": [ + 0.09887817400158383, + 0.45148426100058714, + 0.5206532600022911, + 0.5947030140014249, + 0.6826307560004352, + 0.7625618439997197, + 0.8505774090008345, + 0.9556595620015287, + 1.0668850020010723, + 1.1753313680019346, + 1.2827668030004133, + 1.382749689000775, + 1.486129486002028, + 1.573589070001617, + 1.66888444900178, + 1.7782992669999658, + 1.900071347001358, + 2.0289332770007604, + 2.1480568860024505, + 2.262698457001534, + 2.3664227059998666, + 2.476723931002198, + 2.6031099489991902, + 2.7304616719993646, + 2.8638792279998597, + 3.008441705002042, + 3.1362756170019566, + 3.2564031690017146, + 3.3628542320002452, + 3.452490553001553, + 3.5518499569989217, + 3.655955365000409, + 3.769142250999721, + 3.894313474000228, + 4.020980162000342, + 4.131426538002415, + 4.237025348000316, + 4.3424395469992305, + 4.458939865002321, + 4.582292143000814, + 4.6925186200023745, + 4.814661319000152, + 4.9464083999992, + 5.075429106000229, + 5.187987168999825, + 5.312883044000046, + 5.440164906001883, + 5.555141567001556, + 5.669354754001688, + 5.772394579002139, + 5.853596747001575, + 5.958843963002437, + 6.0740399659989635, + 6.177788150002016, + 6.300556933001644, + 6.435702270002366, + 6.580747586001962, + 6.718620716001169, + 6.855585322002298 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 112.87247449945426, + "p95": 144.56247700218228, + "max": 352.6060869990033 + } + }, + { + "start": 29069.516690908, + "first": 29069.805623893, + "last": 29078.171695938, + "end": 29078.171770594, + "elapsed_s": 8.655079685999226, + "chunk_offsets_s": [ + 0.28893298500042874, + 0.45127547800075263, + 0.5199567210001987, + 0.5940489720014739, + 0.6818035920005059, + 0.7623571690019162, + 0.8502238520013634, + 0.9552580710005714, + 1.066194713999721, + 1.175167927001894, + 1.2821829169988632, + 1.3817649699994945, + 1.4857142789987847, + 1.5729978439994738, + 1.6682693680013472, + 1.777711080001609, + 1.8990964440017706, + 2.028062860001228, + 2.1476971590018366, + 2.2617386630008696, + 2.3662723209999967, + 2.4766952530007984, + 2.6023989179993805, + 2.7300167419998616, + 2.863262434999342, + 3.0086541370001214, + 3.135681735999242, + 3.256126377000328, + 3.362051309999515, + 3.4517355659991154, + 3.551914036001108, + 3.655870364000293, + 3.7681095990010363, + 3.894273530000646, + 4.021004524998716, + 4.131076120000216, + 4.236867916999472, + 4.34208791600031, + 4.458862463001424, + 4.581964542998321, + 4.692438355999911, + 4.814357685998402, + 4.9463521520010545, + 5.074779020000278, + 5.187088163998851, + 5.311165014001745, + 5.439365686001111, + 5.555431573000533, + 5.668524339998839, + 5.771988681000948, + 5.853297050001856, + 5.957976197001699, + 6.073591488999227, + 6.176998413000547, + 6.300374793001538, + 6.43558103399846, + 6.580572913000651, + 6.718440926000767, + 6.85627478600145, + 6.945226296000328, + 7.02792762199897, + 7.106558916999347, + 7.192595333999634, + 7.291137975000311, + 7.391591138999502, + 7.471281157999329, + 7.565036558000429, + 7.651342393000959, + 7.733658984998328, + 7.802018029000465, + 7.8738909130006505, + 7.964788721001241, + 8.03438499800177, + 8.112772322998353, + 8.183540449001157, + 8.25018687299962, + 8.324649714999396, + 8.40020092800114, + 8.464553729998443, + 8.508977081000921, + 8.555540763001773, + 8.603083021000202, + 8.655005029999302 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 104.85668299952522, + "p95": 137.83386000068276, + "max": 162.34249300032388 + } + }, + { + "start": 29069.516362465, + "first": 29069.89950965, + "last": 29077.249267473, + "end": 29077.249813824, + "elapsed_s": 7.733451358999446, + "chunk_offsets_s": [ + 0.38314718499896117, + 0.45136340799945174, + 0.520715073998872, + 0.594677770997805, + 0.68234285699873, + 0.7627905599983933, + 0.8500244559982093, + 0.9554370869991544, + 1.0663078899997345, + 1.1751150009986304, + 1.2827021509983751, + 1.382307451000088, + 1.4862260139998398, + 1.5737164559977828, + 1.6684687999986636, + 1.7778672940003162, + 1.8997237449984823, + 2.0285425699985353, + 2.1481298629987577, + 2.262131955998484, + 2.366762474997813, + 2.4770944459996826, + 2.602825499998289, + 2.7304255130002275, + 2.863987447999534, + 3.0085961799995857, + 3.1358691259993066, + 3.256801743998949, + 3.3627298639985383, + 3.452394596999511, + 3.552050515998417, + 3.65605795999727, + 3.768721206997725, + 3.8945218039989413, + 4.020992514997488, + 4.13132528999995, + 4.23658732399781, + 4.342760013998486, + 4.458930554999824, + 4.582564082000317, + 4.692682760000025, + 4.814764491999085, + 4.946792837999965, + 5.074939529000403, + 5.18769940799757, + 5.312482524997904, + 5.4398672619972785, + 5.5550953399979335, + 5.6691236259975994, + 5.772559042998182, + 5.8537942609982565, + 5.958428301997628, + 6.073791537997749, + 6.177535260998411, + 6.3006080979976105, + 6.4359464129993285, + 6.58093781199932, + 6.718580021999514, + 6.8565504159996635, + 6.945424712997919, + 7.0281746789987665, + 7.106723531996977, + 7.192806828999892, + 7.291402044000279, + 7.392153190998215, + 7.4716815699976, + 7.565214938000281, + 7.651994419000403, + 7.732905007997033 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 109.10280250027427, + "p95": 137.6422100001946, + "max": 144.99139899999136 + } + }, + { + "start": 29069.516088142, + "first": 29069.710974823, + "last": 29077.481774495, + "end": 29077.481941747, + "elapsed_s": 7.965853604997392, + "chunk_offsets_s": [ + 0.19488668099802453, + 0.4513488859993231, + 0.5203087259978929, + 0.5944102499997825, + 0.6819667909985583, + 0.7632810449977114, + 0.851182197999151, + 0.9558140069966612, + 1.0666843459985103, + 1.1756701429985696, + 1.2830760990000272, + 1.3826959969992458, + 1.4864172769994184, + 1.5738646129975677, + 1.6687948619983217, + 1.7784544969981653, + 1.8996568249967822, + 2.02889493299881, + 2.1480972379977175, + 2.262261646999832, + 2.3672507279989077, + 2.477096000999154, + 2.602759197998239, + 2.730510672998207, + 2.8637111819989514, + 3.009114378997765, + 3.136227645998588, + 3.256653019998339, + 3.3628102080001554, + 3.45210831499935, + 3.5522319749979943, + 3.6562023559999943, + 3.7686653419987124, + 3.8945147529993847, + 4.0215429669988225, + 4.131855659998109, + 4.237413218997972, + 4.342643945998134, + 4.459855715998856, + 4.582682031999866, + 4.69284613399941, + 4.814872576000198, + 4.946817842999735, + 5.075462580000021, + 5.187769836997177, + 5.313293305996922, + 5.439911705998384, + 5.555758714999683, + 5.669063579996873, + 5.772661592000077, + 5.853844604996993, + 5.958839924998756, + 6.0738207759968645, + 6.177921668997442, + 6.301071288999083, + 6.436127372999181, + 6.581098321999889, + 6.719151732999308, + 6.856274587997177, + 6.945611556999211, + 7.028225979996932, + 7.107212658000208, + 7.1932451729990134, + 7.291500704999635, + 7.392257511997741, + 7.471629799998482, + 7.565250296000158, + 7.651698278998083, + 7.733298518000083, + 7.802368014999956, + 7.874418622999656, + 7.96568635299991 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 108.98579700005939, + "p95": 138.0534109994187, + "max": 256.46220500129857 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 132.8, + "per_stream_tps": 30.3, + "ttft_s_max": 0.49, + "request": { + "ttft_s": 0.286, + "decode_tps": 30.309, + "prefill_tps": 48.95, + "itl_mean_ms": 32.994, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.056, + "round_gap_p50_ms": 125.6, + "round_gap_p95_ms": 150.497, + "tokens_per_chunk": 3.71 + }, + "pooled": { + "aggregate_prefill_tps": 146.766, + "itl_mean_ms": 33.201, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.963, + "round_gap_p50_ms": 125.455, + "round_gap_p95_ms": 150.517, + "tokens_per_chunk": 3.657 + }, + "timing_valid": true, + "end_to_end_tps": 131.92695233342502, + "makespan_s": 9.702338888000668, + "steady_tps": 151.0, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 132.8390122314465, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 9.694614144002117, + "end_to_end_tps": 132.03207275576955, + "timing_valid": true, + "steady_tps": 152.5602337672633, + "per_stream_tps": [ + 34.9, + 26.6, + 29.7, + 29.9, + 30.7 + ], + "ttft_s": [ + 0.38, + 0.1, + 0.19, + 0.48, + 0.28 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.3791816970006039, + "decode_tps": 34.901415073944634, + "prefill_tps": 36.92161333403628, + "itl_mean_ms": 28.65213338431489, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.79627500116476, + "round_gap_p50_ms": 127.61297499673674, + "round_gap_p95_ms": 154.8516059992835, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09590855100032059, + "decode_tps": 26.567802446289303, + "prefill_tps": 145.97238571515072, + "itl_mean_ms": 37.639545160787996, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.0560300021898, + "round_gap_p50_ms": 120.79665100100101, + "round_gap_p95_ms": 148.61296299932292, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18906434399832506, + "decode_tps": 29.719430118663198, + "prefill_tps": 74.04886455017677, + "itl_mean_ms": 33.64802070588898, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.44633100234205, + "round_gap_p50_ms": 125.16434699864476, + "round_gap_p95_ms": 149.4926609993854, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.47585099400021136, + "decode_tps": 29.916038434906696, + "prefill_tps": 29.42097458347178, + "itl_mean_ms": 33.426885788232504, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.8204700013157, + "round_gap_p50_ms": 122.56888599949889, + "round_gap_p95_ms": 148.71525099806604, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28263194000101066, + "decode_tps": 30.66111002062325, + "prefill_tps": 49.53438737302634, + "itl_mean_ms": 32.61460525491024, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.4758320018591, + "round_gap_p50_ms": 126.39033999948879, + "round_gap_p95_ms": 149.78590800092206, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 33.19623805882692, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.96261999959825, + "round_gap_p50_ms": 125.1562039979035, + "round_gap_p95_ms": 149.77743499912322, + "round_gap_mean_ms": 122.68174934783863, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 146.7661280858435 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 29078.173323076, + "first": 29078.552504773, + "last": 29085.858798786, + "end": 29085.858978978, + "elapsed_s": 7.685655902001599, + "chunk_offsets_s": [ + 0.3791816970006039, + 0.5471654690009018, + 0.6189497689993004, + 0.70283448400005, + 0.7982494489988312, + 0.8848168279982929, + 0.9818181869995897, + 1.0962219899993215, + 1.216783887000929, + 1.3360450969994417, + 1.4555423839992727, + 1.563929907999409, + 1.6808496849989751, + 1.7887934610007505, + 1.8920843359992432, + 2.014777928001422, + 2.142559831998369, + 2.2759640840013162, + 2.411716045000503, + 2.5386551129995496, + 2.664791896000679, + 2.7964224360002845, + 2.9406265409998014, + 3.085764650000783, + 3.2257132539998565, + 3.38056485999914, + 3.5159637270007806, + 3.647575245999178, + 3.774375593999139, + 3.8867417039982683, + 4.01433299099881, + 4.139591905000998, + 4.268013706001511, + 4.410598096001195, + 4.548360075001256, + 4.672222100998624, + 4.792254504998709, + 4.90938758899938, + 5.04008804499972, + 5.180225177999091, + 5.308650259998103, + 5.446151664000354, + 5.5839479390015185, + 5.716435481001099, + 5.844383581999864, + 5.980346411000937, + 6.12019522600167, + 6.242535582001437, + 6.370148556998174, + 6.507387434998236, + 6.605540691998613, + 6.716840434000915, + 6.834218066000176, + 6.951260298999841, + 7.087833651999972, + 7.22931720699853, + 7.386882377999427, + 7.535227657001087, + 7.685475710000901 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 127.69743949684198, + "p95": 154.8516059992835, + "max": 167.9837720002979 + } + }, + { + "start": 29078.173737899, + "first": 29078.26964645, + "last": 29087.867730466, + "end": 29087.86784296, + "elapsed_s": 9.6941050610003, + "chunk_offsets_s": [ + 0.09590855100032059, + 0.5461422820008011, + 0.6188068550000025, + 0.7023377760015137, + 0.7970264620016678, + 0.8837753590014472, + 0.9813301940012025, + 1.095349551000254, + 1.2163010580006812, + 1.3355776839998725, + 1.4550950689990714, + 1.5637840679992223, + 1.6804692410005373, + 1.7881324869995296, + 1.8917168910011242, + 2.014305348999187, + 2.142102939000324, + 2.2750913689997105, + 2.410747103000176, + 2.538345599998138, + 2.6641812410016428, + 2.7955733829985547, + 2.9401689609985624, + 3.0852416529996844, + 3.225574598000094, + 3.3802449379982136, + 3.5157406270009233, + 3.647110468999017, + 3.77415310500146, + 3.8863759800005937, + 4.01396897900122, + 4.139125182999123, + 4.267645790998358, + 4.410035862001678, + 4.547886584998196, + 4.671629381999082, + 4.792426033000083, + 4.909635975000128, + 5.039476672998717, + 5.179418656000053, + 5.308986574000301, + 5.4456149020006706, + 5.583577522000269, + 5.716421877998073, + 5.84423483700084, + 5.980192300998169, + 6.1200006079998275, + 6.242371512998943, + 6.369451450998895, + 6.507507481001085, + 6.605045126998448, + 6.71632141200098, + 6.834067489999143, + 6.950179591000051, + 7.087085130999185, + 7.228985707999527, + 7.386152003000461, + 7.534764965999784, + 7.684542400998907, + 7.801824761001626, + 7.910130592001224, + 8.016691715998604, + 8.12897263099876, + 8.253333272998134, + 8.371970896998391, + 8.477175377000094, + 8.597995746000379, + 8.686468315001548, + 8.768871600001148, + 8.838384628001222, + 8.911020744999405, + 8.999922302999039, + 9.065508240000781, + 9.142594865999854, + 9.21107706100156, + 9.27880963500138, + 9.354393164998328, + 9.434248590998322, + 9.502017070000875, + 9.54620645400064, + 9.593068990001484, + 9.639643182999862, + 9.69399256700126 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 120.80851000064285, + "p95": 148.61296299932292, + "max": 450.2337310004805 + } + }, + { + "start": 29078.173384013, + "first": 29078.362448357, + "last": 29086.942693637, + "end": 29086.942862765, + "elapsed_s": 8.769478752001305, + "chunk_offsets_s": [ + 0.18906434399832506, + 0.5466129890010052, + 0.6192080549990351, + 0.7030383240016818, + 0.7975626650004415, + 0.8848595220006246, + 0.9819732579999254, + 1.0963182500017865, + 1.2169763099991542, + 1.3361424290014838, + 1.455325016999268, + 1.5640352980008174, + 1.6810132009995868, + 1.7888101239987009, + 1.8926083149999613, + 2.0148274499988474, + 2.142691287001071, + 2.2761097219990916, + 2.4120496950017696, + 2.5392085300009057, + 2.6644195020016923, + 2.7957220829994185, + 2.9407316940014425, + 3.085878772999422, + 3.226127770001767, + 3.380285687999276, + 3.516276640999422, + 3.647566789000848, + 3.774891850000131, + 3.8870383809990017, + 4.014675872000225, + 4.13984021899887, + 4.268354122999881, + 4.410602479998488, + 4.54904881100083, + 4.6718825999996625, + 4.792255196000042, + 4.909510362998844, + 5.040078951999021, + 5.180260844001168, + 5.309028640000179, + 5.44614247500067, + 5.584092965000309, + 5.717010601001675, + 5.844542817998445, + 5.980858328999602, + 6.120956580998609, + 6.242173473001458, + 6.369589163001365, + 6.5072649020003155, + 6.605258859999594, + 6.716604771998391, + 6.834814133999316, + 6.950709567001468, + 7.087954955000896, + 7.22940632999962, + 7.3865722400005325, + 7.535065428000962, + 7.6845580890003475, + 7.801977520000946, + 7.910532158999558, + 8.01669431999835, + 8.129064689001098, + 8.25377627600028, + 8.372569224000472, + 8.477282060001016, + 8.598520335999638, + 8.6868765170002, + 8.769309624000016 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 125.18765949971566, + "p95": 149.4926609993854, + "max": 357.54864500268013 + } + }, + { + "start": 29078.174327108, + "first": 29078.650178102, + "last": 29087.174033978, + "end": 29087.174151743, + "elapsed_s": 8.999824634996912, + "chunk_offsets_s": [ + 0.47585099400021136, + 0.546337804000359, + 0.6180566109978827, + 0.7027765599996201, + 0.797521610998956, + 0.883348847997695, + 0.9811552840001241, + 1.0945619859994622, + 1.2161553619989718, + 1.3352755419982714, + 1.4543336169990653, + 1.563150134999887, + 1.6798001770002884, + 1.7880167919975065, + 1.8914173579978524, + 2.0139862439973513, + 2.142094200997235, + 2.2755023229983635, + 2.4108410209992144, + 2.5379704179977125, + 2.663644847998512, + 2.794648827999481, + 2.9402134269985254, + 3.0851781700002903, + 3.2255617549999442, + 3.3794376549994922, + 3.515268865998223, + 3.646758155999123, + 3.774037450999458, + 3.88601132599797, + 4.013908359997004, + 4.139061171998037, + 4.267704568999761, + 4.409106781000446, + 4.548319823999918, + 4.670863031999033, + 4.7919878419997985, + 4.909238767999341, + 5.039356422999845, + 5.1791218599973945, + 5.307782914998825, + 5.445417921997432, + 5.583238391998748, + 5.71622118400046, + 5.843716459999996, + 5.979660463999608, + 6.120552096999745, + 6.241934054000012, + 6.369383585999458, + 6.506167567000375, + 6.605275912999787, + 6.716075887998159, + 6.83402570799808, + 6.950331369000196, + 7.086317551998945, + 7.227979991999746, + 7.3858383299993875, + 7.5345535809974535, + 7.684624834997521, + 7.80152796299808, + 7.909889020000264, + 8.015921463997074, + 8.12799850599913, + 8.253074233998632, + 8.371133649998228, + 8.476740993999556, + 8.59784693599795, + 8.685745353999664, + 8.769013682998775, + 8.837970307999058, + 8.91086898499998, + 8.9997068699995 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 122.56888599949889, + "p95": 148.71525099806604, + "max": 157.85833799964166 + } + }, + { + "start": 29078.173228816, + "first": 29078.455860756, + "last": 29086.772585096, + "end": 29086.772738815, + "elapsed_s": 8.599509999003203, + "chunk_offsets_s": [ + 0.28263194000101066, + 0.547339278000436, + 0.6191075250026188, + 0.703582620000816, + 0.7979761800015694, + 0.8848564340005396, + 0.9821950690020458, + 1.096095573000639, + 1.2166151119999995, + 1.3364232110034209, + 1.4555184780001582, + 1.5639692260010634, + 1.6808197340033075, + 1.7888421500028926, + 1.8921055370010436, + 2.0147371840030246, + 2.1425260590003745, + 2.275852216000203, + 2.4114890490018297, + 2.538703529000486, + 2.665093868999975, + 2.7968158399999083, + 2.9411102260019106, + 3.0857018220012833, + 3.2260243730015645, + 3.3807232400031353, + 3.5160150060000888, + 3.6479070570021577, + 3.7747020080023503, + 3.886946689002798, + 4.014535001002514, + 4.139948956002627, + 4.268461117000697, + 4.410656631000165, + 4.5491324630020245, + 4.672421109000425, + 4.792242490002536, + 4.91028838100101, + 5.040333869001188, + 5.180374576000759, + 5.309243827003229, + 5.4463914960033435, + 5.583735826003249, + 5.7166060020026634, + 5.844833094000933, + 5.980502653001167, + 6.120857666002848, + 6.242385094003112, + 6.370399675000954, + 6.507780458003253, + 6.605351095000515, + 6.716989001000911, + 6.834649716001877, + 6.951146690000314, + 7.0869763900009275, + 7.228816236001876, + 7.386884177001775, + 7.535721756001294, + 7.685507664002216, + 7.802032008999959, + 7.910737975002121, + 8.016791465001006, + 8.129284472001018, + 8.253888447001373, + 8.372863129003235, + 8.477362894001999, + 8.599356280003121 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 126.59264549984073, + "p95": 149.78590800092206, + "max": 264.70733799942536 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 132.7403821918092, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 9.702338888000668, + "end_to_end_tps": 131.92695233342502, + "timing_valid": true, + "steady_tps": 150.96227250750815, + "per_stream_tps": [ + 34.8, + 27.1, + 30.7, + 28.6, + 30.3 + ], + "ttft_s": [ + 0.38, + 0.29, + 0.48, + 0.1, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.37973866000174894, + "decode_tps": 34.84853328878399, + "prefill_tps": 36.86746037376211, + "itl_mean_ms": 28.695612286266588, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.0586289995117, + "round_gap_p50_ms": 129.1798940001172, + "round_gap_p95_ms": 151.94853600041824, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28600672699758434, + "decode_tps": 27.081113521663262, + "prefill_tps": 48.949897601947825, + "itl_mean_ms": 36.92610347059991, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.06610499866656, + "round_gap_p50_ms": 120.87303700172924, + "round_gap_p95_ms": 147.00846300183912, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.47531028099911055, + "decode_tps": 30.701414295726597, + "prefill_tps": 29.4544438857324, + "itl_mean_ms": 32.571789376465055, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.2187250017305, + "round_gap_p50_ms": 124.96678000024986, + "round_gap_p95_ms": 147.73371200135443, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09661736699854373, + "decode_tps": 28.598283198878494, + "prefill_tps": 144.90148546700735, + "itl_mean_ms": 34.96713397254615, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.61654400150292, + "round_gap_p50_ms": 125.70849300027476, + "round_gap_p95_ms": 151.43370700025116, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19196825100152637, + "decode_tps": 30.308800306226225, + "prefill_tps": 72.92872611465675, + "itl_mean_ms": 32.993717662740146, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.48004399894853, + "round_gap_p50_ms": 127.48059099976672, + "round_gap_p95_ms": 150.49672399982228, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 33.23087135372357, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.2187250017305, + "round_gap_p50_ms": 126.43761099752737, + "round_gap_p95_ms": 150.51672399931704, + "round_gap_mean_ms": 122.80974195941319, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 147.0900619356566 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 29087.870162762, + "first": 29088.249901422, + "last": 29095.567282555, + "end": 29095.567598963, + "elapsed_s": 7.697436201000528, + "chunk_offsets_s": [ + 0.37973866000174894, + 0.5446291799999017, + 0.6182019930020033, + 0.7093860550012323, + 0.8055502349998278, + 0.8924154220003402, + 0.9923208309992333, + 1.1072009870003967, + 1.2278294269999606, + 1.344675971999095, + 1.4565934749989538, + 1.5622842890006723, + 1.6849308790006035, + 1.794944919001864, + 1.9004234460007865, + 2.025407439999981, + 2.1542715659998066, + 2.2860611750002136, + 2.4145224080020853, + 2.5420335740018345, + 2.672971903000871, + 2.804700665001292, + 2.95127743500052, + 3.0985706920000666, + 3.2397619080002187, + 3.3902786319995357, + 3.523414973002218, + 3.6583470580007997, + 3.790475312001945, + 3.90393352600222, + 4.033265271002165, + 4.1596961630020814, + 4.289793353000277, + 4.426698560000659, + 4.563551711002219, + 4.69430384700172, + 4.8156946420021995, + 4.932072687999607, + 5.064654556001187, + 5.204375351000635, + 5.333555245000753, + 5.463583382999786, + 5.600213257999712, + 5.739143917999172, + 5.87032221499976, + 6.007152729001973, + 6.145024557001307, + 6.266257490002317, + 6.387216766001075, + 6.5144582009997976, + 6.614052641001763, + 6.734935410000617, + 6.850136017001205, + 6.9657050509995315, + 7.102763679999043, + 7.2439672440013965, + 7.395915780001815, + 7.540354600001592, + 7.697119792999729 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 129.25581950003107, + "p95": 151.94853600041824, + "max": 164.89051999815274 + } + }, + { + "start": 29087.869925241, + "first": 29088.155931968, + "last": 29097.572088353, + "end": 29097.572157733, + "elapsed_s": 9.702232491999894, + "chunk_offsets_s": [ + 0.28600672699758434, + 0.5452918999981193, + 0.6185314829999697, + 0.7101188870001351, + 0.8058949549995305, + 0.8927477850011201, + 0.9919581680005649, + 1.107774435000465, + 1.2275432809983613, + 1.3444051330006914, + 1.456930119999015, + 1.5625675689989293, + 1.6848738669978047, + 1.794473874000687, + 1.9009772399986105, + 2.025700610000058, + 2.1543649819977873, + 2.2863788409995323, + 2.4149311920009495, + 2.542402029001096, + 2.6734748859998945, + 2.8048553299995547, + 2.951235166998231, + 3.09824363000007, + 3.239848321998579, + 3.3904630309989443, + 3.523488186998293, + 3.659166869998444, + 3.790750781998213, + 3.904108508999343, + 4.033335415999318, + 4.160108887001115, + 4.289995289000217, + 4.426796800999, + 4.5638629059976665, + 4.694632842998544, + 4.8159869449991675, + 4.93220717100121, + 5.064993336000043, + 5.204370733998076, + 5.3339526140007365, + 5.4635634849983035, + 5.599928084000567, + 5.739267175998975, + 5.8708772210011375, + 6.007609156000399, + 6.145378216999234, + 6.266821174998768, + 6.387587387998792, + 6.514835732999927, + 6.61458244899768, + 6.73545548599941, + 6.850971610998386, + 6.9659943179976835, + 7.102875586999289, + 7.244555942997977, + 7.396084818999952, + 7.540665941000043, + 7.697149094998167, + 7.817676512000617, + 7.9259868699991785, + 8.03226570699917, + 8.14685012100017, + 8.268479245998606, + 8.382233398999233, + 8.483224571999017, + 8.605315574000997, + 8.697332940999331, + 8.781709047998447, + 8.849566757999128, + 8.923126858000614, + 9.013198414999351, + 9.077578278000146, + 9.155770008001127, + 9.224056070001097, + 9.289861032000772, + 9.364355599998817, + 9.439149996000197, + 9.503604015000747, + 9.548908264998317, + 9.595835896998324, + 9.644017205999262, + 9.702163112000562 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 121.11356950117624, + "p95": 147.00846300183912, + "max": 259.285173000535 + } + }, + { + "start": 29087.870407473, + "first": 29088.345717754, + "last": 29096.651524045, + "end": 29096.652085011, + "elapsed_s": 8.781677537997894, + "chunk_offsets_s": [ + 0.47531028099911055, + 0.5448638279995066, + 0.6182459150004433, + 0.7096981969989429, + 0.805364739997458, + 0.8922191589990689, + 0.9916823989988188, + 1.1070231339981547, + 1.227340612000262, + 1.3440689079980075, + 1.4564138889982132, + 1.5621298609985388, + 1.68433320099939, + 1.7944177579993266, + 1.9006499989991426, + 2.0256167789993924, + 2.1544090530005633, + 2.285712254997634, + 2.414324345998466, + 2.541752015000384, + 2.6721871409972664, + 2.8041983369985246, + 2.950717824998719, + 3.0984515370000736, + 3.239442052999948, + 3.3901664379991416, + 3.522942438998143, + 3.6582962139982556, + 3.7900529010003083, + 3.90354370699788, + 4.033058070999687, + 4.159495681997214, + 4.289329273997282, + 4.425569480998092, + 4.563424357998883, + 4.694115008998779, + 4.815823397999338, + 4.931897503000073, + 5.0644614929988165, + 5.204199140000128, + 5.333504150999943, + 5.463132482997025, + 5.59949028399933, + 5.739117404998979, + 5.86985096799981, + 6.007005670999206, + 6.144843942998705, + 6.266484148000018, + 6.387161958999059, + 6.513980634998006, + 6.613759928997752, + 6.734845523998956, + 6.850040198998613, + 6.965398109998205, + 7.102616834999935, + 7.244022268998378, + 7.3953028510004515, + 7.5401562489969365, + 7.696945518997381, + 7.817304312000488, + 7.92603817899726, + 8.03135565699995, + 8.146068219997687, + 8.268389551998553, + 8.382040593998681, + 8.482690807999461, + 8.604722681000567, + 8.697096088999388, + 8.7811165719977 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 125.70219549888861, + "p95": 147.73371200135443, + "max": 156.78927000044496 + } + }, + { + "start": 29087.870255309, + "first": 29087.966872676, + "last": 29096.883491839, + "end": 29096.883643516, + "elapsed_s": 9.01338820700039, + "chunk_offsets_s": [ + 0.09661736699854373, + 0.5444767749977473, + 0.6175443270003598, + 0.7097261199996865, + 0.8052479129983112, + 0.8920084849996783, + 0.9915756179980235, + 1.1070481440001458, + 1.2270945310010575, + 1.3438523490003718, + 1.4561774030007655, + 1.5620096669990744, + 1.6846625459984352, + 1.7942566809979326, + 1.8999741449988505, + 2.0256826379991253, + 2.153538448998006, + 2.2861458910010697, + 2.4142052739989595, + 2.541643866999948, + 2.672560490998876, + 2.804277944000205, + 2.9506074729979446, + 3.09912324600009, + 3.2397374479987775, + 3.3911711549990287, + 3.5228761799990025, + 3.658511556997837, + 3.7900784989979, + 3.9034778169989295, + 4.032855179000762, + 4.159523190999607, + 4.289564075999806, + 4.425445361001039, + 4.563326282997878, + 4.6943758060006076, + 4.815378637998947, + 4.931082188999426, + 5.064296700998966, + 5.204196056998626, + 5.333375758000329, + 5.463009381997836, + 5.600327477000974, + 5.738858538999921, + 5.870772550999391, + 6.006854244998976, + 6.144470789000479, + 6.265896416000032, + 6.386787858999014, + 6.514595013999497, + 6.614086076999229, + 6.734657495999272, + 6.850603322000097, + 6.965385000999959, + 7.102392652999697, + 7.24371004799832, + 7.396092191000207, + 7.540174927999033, + 7.697307217000343, + 7.817281183997693, + 7.9266112679979415, + 8.031860609000432, + 8.145631592000427, + 8.268019250001089, + 8.381514750999486, + 8.483241745998384, + 8.60471504099769, + 8.697348318997683, + 8.78086379099841, + 8.84915890299817, + 8.923023407998699, + 9.013236529997812 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 125.70849300027476, + "p95": 151.43370700025116, + "max": 447.85940799920354 + } + }, + { + "start": 29087.869818845, + "first": 29088.061787096, + "last": 29096.4751851, + "end": 29096.475364796, + "elapsed_s": 8.60554595099893, + "chunk_offsets_s": [ + 0.19196825100152637, + 0.5456518809987756, + 0.6180351770017296, + 0.7099165420004283, + 0.805778737001674, + 0.892975357000978, + 0.9926007139983994, + 1.1076622549990134, + 1.2277529730017704, + 1.3448016649999772, + 1.456512800999917, + 1.5627968029984913, + 1.6851933210018615, + 1.7950941019989841, + 1.9006208259997948, + 2.0259637519993703, + 2.1540799229987897, + 2.2868071500015503, + 2.414805669999623, + 2.54228626099939, + 2.6731616409997514, + 2.8051074510003673, + 2.9512323189992458, + 3.0993218629992043, + 3.2402191100009077, + 3.39071583400073, + 3.523479822000809, + 3.658844566998596, + 3.7911116409995884, + 3.9043295060000673, + 4.033501196001453, + 4.160120488999382, + 4.29005614000198, + 4.426557858998422, + 4.56407362200116, + 4.6945592730007775, + 4.816221764001966, + 4.931887287999416, + 5.065115060999233, + 5.20435337500021, + 5.334000308001123, + 5.464076276999549, + 5.600817720998748, + 5.739857783999469, + 5.870916127001692, + 6.007674814001803, + 6.145154858000751, + 6.26685760500186, + 6.387355997001578, + 6.515379701999336, + 6.614258038000116, + 6.735492398001952, + 6.850974261000374, + 6.966205472999718, + 7.103045223000663, + 7.244564136999543, + 7.39611112700004, + 7.540811692000716, + 7.697109276999981, + 7.8174996500019915, + 7.926309393998963, + 8.031706501998997, + 8.14638833799836, + 8.26888019800026, + 8.38288832300168, + 8.483107258001837, + 8.605366255000263 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 127.7395554989198, + "p95": 150.49672399982228, + "max": 353.6836299972492 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 132.75275691726887, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 9.705036087998451, + "end_to_end_tps": 131.89028751607506, + "timing_valid": true, + "steady_tps": 150.75964483143628, + "per_stream_tps": [ + 33.6, + 27.7, + 30.4, + 28.9, + 30.7 + ], + "ttft_s": [ + 0.1, + 0.49, + 0.39, + 0.2, + 0.3 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.10051521100103855, + "decode_tps": 33.58562070576592, + "prefill_tps": 139.28240174370572, + "itl_mean_ms": 29.77464697647591, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.58569999886095, + "round_gap_p50_ms": 128.43427000188967, + "round_gap_p95_ms": 151.94619199974113, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.48537076600041473, + "decode_tps": 27.66004396369583, + "prefill_tps": 28.843929178849717, + "itl_mean_ms": 36.15323248627201, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.55008499740507, + "round_gap_p50_ms": 118.91616099819657, + "round_gap_p95_ms": 147.62661699933233, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3890709869992861, + "decode_tps": 30.39601076441508, + "prefill_tps": 35.983150807453264, + "itl_mean_ms": 32.899054015690446, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.98410000203876, + "round_gap_p50_ms": 125.59959600184811, + "round_gap_p95_ms": 151.10137199735618, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.200266747000569, + "decode_tps": 28.924068412090023, + "prefill_tps": 69.90676290338017, + "itl_mean_ms": 34.573282905872546, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.33289700071327, + "round_gap_p50_ms": 125.45499800035032, + "round_gap_p95_ms": 151.03295800145133, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2954363610006112, + "decode_tps": 30.67114734891338, + "prefill_tps": 47.38753196317307, + "itl_mean_ms": 32.603931917644026, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.8164269994595, + "round_gap_p50_ms": 126.17327399857459, + "round_gap_p95_ms": 150.93602300112252, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 33.20082966039099, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.54035799906706, + "round_gap_p50_ms": 125.45499800035032, + "round_gap_p95_ms": 150.93602300112252, + "round_gap_mean_ms": 122.6987183101406, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 144.066821682671 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 29097.573595525, + "first": 29097.674110736, + "last": 29105.266645715, + "end": 29105.266861787, + "elapsed_s": 7.693266262001998, + "chunk_offsets_s": [ + 0.10051521100103855, + 0.5548697539998102, + 0.6280935579998186, + 0.7124897970024904, + 0.8087445700002718, + 0.8957416170014767, + 0.9987795850029215, + 1.11672530800206, + 1.236776252000709, + 1.3549160090005898, + 1.4688964250017307, + 1.57378460900145, + 1.6928841750013817, + 1.8008101480008918, + 1.9063683550011774, + 2.0350870890033548, + 2.1672344409998914, + 2.298713760999817, + 2.427864220000629, + 2.5545317530013563, + 2.680959100001928, + 2.810274698000285, + 2.958361405002506, + 3.108693285001209, + 3.248233643000276, + 3.400179835000017, + 3.532575318000454, + 3.6644221560018195, + 3.789529500001663, + 3.90157431300031, + 4.038074189000326, + 4.16681410000092, + 4.29524837000281, + 4.431710058001045, + 4.568331667000166, + 4.693127754002489, + 4.813253266001993, + 4.92838488200141, + 5.067970582000271, + 5.207798463001382, + 5.338644745002966, + 5.4697523710019595, + 5.606043317002332, + 5.738449428001331, + 5.865572874001373, + 6.005903007000597, + 6.14880528300273, + 6.270085793999897, + 6.392418266001187, + 6.51923648400043, + 6.618935839000187, + 6.731218208002247, + 6.846287471002142, + 6.965206971002772, + 7.1062475860017, + 7.24631891700119, + 7.4003589210005885, + 7.54434042000139, + 7.6930501900023955 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 128.57650200203352, + "p95": 151.94619199974113, + "max": 454.35454299877165 + } + }, + { + "start": 29097.573986, + "first": 29098.059356766, + "last": 29107.27843105, + "end": 29107.278507213, + "elapsed_s": 9.70452121299968, + "chunk_offsets_s": [ + 0.48537076600041473, + 0.5545640199998161, + 0.6279702700012422, + 0.7126346080003714, + 0.8092109560020617, + 0.8955324500020652, + 0.998684628000774, + 1.1162869820000196, + 1.2365248130008695, + 1.35482924399912, + 1.4683993620019464, + 1.5731704130012076, + 1.6920865739994042, + 1.7998974340007408, + 1.9052364560011483, + 2.034366112002317, + 2.166749844000151, + 2.298461717000464, + 2.4275791860018217, + 2.554310584000632, + 2.6806117560008715, + 2.8096808709997276, + 2.95730748799906, + 3.109441707001679, + 3.247991791999084, + 3.399951440998848, + 3.532422526001028, + 3.6640052309994644, + 3.7896715850001783, + 3.9015881049999734, + 4.037292040000466, + 4.166328061000968, + 4.294718334000208, + 4.43119323800056, + 4.567971373002365, + 4.692406574002234, + 4.812405595999735, + 4.928255378999893, + 5.0676634080009535, + 5.207709425001667, + 5.338631611000892, + 5.469562540001789, + 5.606011715000932, + 5.738689510999393, + 5.865854732001026, + 6.005398867000622, + 6.148243349001859, + 6.269564908001485, + 6.392135922000307, + 6.519181584000762, + 6.618824429999222, + 6.731141139000101, + 6.846969131998776, + 6.964853608002159, + 7.105380778000836, + 7.247215924002376, + 7.399799749000522, + 7.543838906000019, + 7.693376128001546, + 7.810378209000191, + 7.918639297000482, + 8.030141486000502, + 8.147538491000887, + 8.270626274999813, + 8.385295320000296, + 8.48827880799945, + 8.60959949500102, + 8.696903947002284, + 8.778609152999707, + 8.84774003200073, + 8.919902704001288, + 9.016482543000166, + 9.084244696001406, + 9.160437154001556, + 9.229092270001274, + 9.29678416400202, + 9.370759389999876, + 9.44725496600222, + 9.511637766001513, + 9.557786216999375, + 9.604196258002048, + 9.650310822002211, + 9.704445049999777 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 119.45759149784863, + "p95": 147.62661699933233, + "max": 152.58382499814616 + } + }, + { + "start": 29097.574219077, + "first": 29097.963290064, + "last": 29106.352548838, + "end": 29106.35273434, + "elapsed_s": 8.778515262998553, + "chunk_offsets_s": [ + 0.3890709869992861, + 0.5546742320002522, + 0.6283619790010562, + 0.7124966400006087, + 0.8088835560010921, + 0.8952132699996582, + 0.9982443310000235, + 1.1156219490003423, + 1.235678562999965, + 1.3544778169998608, + 1.467548528999032, + 1.572851177999837, + 1.6917648390008253, + 1.7998354560004373, + 1.9056769919989165, + 2.0340099269997154, + 2.1665690680019907, + 2.298850360999495, + 2.4277185419996385, + 2.554298387000017, + 2.6806557170020824, + 2.808723516001919, + 2.9573465940011374, + 3.1084479659984936, + 3.2484320660005324, + 3.4000752739993914, + 3.533488215998659, + 3.663525194999238, + 3.789124791001086, + 3.901447868000105, + 4.037387051001133, + 4.165815342999849, + 4.294191477001732, + 4.430636519999098, + 4.567354245999013, + 4.69228910899983, + 4.812275108000904, + 4.92793560499922, + 5.068189091001841, + 5.206920763001108, + 5.337842428998556, + 5.469307054001547, + 5.60533192100047, + 5.737623983000958, + 5.865268197001569, + 6.005485818001034, + 6.148252802999195, + 6.269773118001467, + 6.392458766000345, + 6.51837184300166, + 6.618165201998636, + 6.730153993001295, + 6.8458283500003745, + 6.964097652999044, + 7.105793142000039, + 7.246039117999317, + 7.399931613999797, + 7.54433869100103, + 7.693332153001393, + 7.809658613001375, + 7.917987737000658, + 8.02955226099948, + 8.14710287500202, + 8.270192241001496, + 8.385329887001717, + 8.487875138998788, + 8.610187029000372, + 8.697191199000372, + 8.77832976100035 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 125.75633650158125, + "p95": 151.10137199735618, + "max": 165.60324500096613 + } + }, + { + "start": 29097.573881378, + "first": 29097.774148125, + "last": 29106.590335266, + "end": 29106.590619741, + "elapsed_s": 9.016738362999604, + "chunk_offsets_s": [ + 0.200266747000569, + 0.5548119539998879, + 0.6279192879992479, + 0.7122921629998018, + 0.8091431799985003, + 0.8953042629982519, + 0.9986906159974751, + 1.11589716600065, + 1.236409992998233, + 1.3547176600004605, + 1.4679540079996514, + 1.5733248680007819, + 1.6928488979974645, + 1.8003245939980843, + 1.9059307120005542, + 2.034680851000303, + 2.166722881000169, + 2.2986991139987367, + 2.4277989590009383, + 2.5550334809995547, + 2.681112395999662, + 2.809188750998146, + 2.9582226339989575, + 3.109255592000409, + 3.248287877999246, + 3.4005095129978145, + 3.5327945979988726, + 3.6640445529992576, + 3.789848988999438, + 3.9018459649996657, + 4.037953188999381, + 4.166619404997618, + 4.295111290000932, + 4.431078409998008, + 4.567965444999572, + 4.693420442999923, + 4.812361622000026, + 4.928492097998969, + 5.0678249949996825, + 5.207408819998818, + 5.338094078997528, + 5.469325172998651, + 5.605509934997826, + 5.737897367998812, + 5.865368665999995, + 6.005788116999611, + 6.148641503998078, + 6.2697568719995616, + 6.3920533359996625, + 6.51876307099883, + 6.618231186999765, + 6.730099977998179, + 6.846631458000047, + 6.964634379997733, + 7.105630658999871, + 7.247028957997827, + 7.399829992999003, + 7.544460413999332, + 7.6932998489974125, + 7.810075031000451, + 7.918398965997767, + 8.030076465998718, + 8.147833463001007, + 8.270823111997743, + 8.385284569998475, + 8.488267593998899, + 8.609614864999457, + 8.69725122899763, + 8.778307136999501, + 8.847727055999712, + 8.919756561997929, + 9.016453887998068 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 125.45499800035032, + "p95": 151.03295800145133, + "max": 354.5452069993189 + } + }, + { + "start": 29097.573471125, + "first": 29097.868907486, + "last": 29106.182910125, + "end": 29106.183915101, + "elapsed_s": 8.61044397599835, + "chunk_offsets_s": [ + 0.2954363610006112, + 0.5550454500007618, + 0.6293645209989336, + 0.7134688469996036, + 0.8090425059999689, + 0.8960920509998687, + 0.9992598860008002, + 1.116891500998463, + 1.2370006710007146, + 1.3555416409981262, + 1.468812983999669, + 1.574028062997968, + 1.692402229000436, + 1.8003425909992075, + 1.9056683769995288, + 2.035137587998179, + 2.1675286749996303, + 2.2992161859983753, + 2.428353409999545, + 2.5547289899986936, + 2.681803579998814, + 2.809949277998385, + 2.958251913998538, + 3.1092732079996495, + 3.2494577199977357, + 3.400393742998858, + 3.533970432999922, + 3.6641630369995255, + 3.7895925249977154, + 3.9018325500001083, + 4.037613817999954, + 4.167129130000831, + 4.295775866998156, + 4.432018136998522, + 4.568330299000081, + 4.693404418998398, + 4.813236934998713, + 4.928853426998103, + 5.068854894998367, + 5.2086713219978265, + 5.338848290997703, + 5.470478606999677, + 5.606336388998898, + 5.738874699000007, + 5.866456425999786, + 6.006152461999591, + 6.148886897997727, + 6.270732623997901, + 6.393347653000092, + 6.519520926998666, + 6.619213089998084, + 6.731552110999473, + 6.84721900400109, + 6.9649878359996364, + 7.105547088998719, + 7.246583825999551, + 7.400163800997689, + 7.544430866000766, + 7.693037444998481, + 7.811154017999797, + 7.919036661998689, + 8.03014943800008, + 8.147527622997586, + 8.271297360999597, + 8.385358826999436, + 8.488946610999847, + 8.609438999999838 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 126.2744269988616, + "p95": 150.93602300112252, + "max": 259.60908900015056 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.5, + "per_stream_tps": 42.5, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.083, + "decode_tps": 42.484, + "prefill_tps": null, + "itl_mean_ms": 23.538, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.018, + "round_gap_p50_ms": 53.501, + "round_gap_p95_ms": 71.372, + "tokens_per_chunk": 2.349 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 23.538, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.018, + "round_gap_p50_ms": 53.501, + "round_gap_p95_ms": 71.372, + "tokens_per_chunk": 2.349 + }, + "timing_valid": true, + "end_to_end_tps": 42.067297228417246, + "makespan_s": 6.0854872279996925, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.431546925030815, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.090930940001272, + "end_to_end_tps": 42.02969997882566, + "timing_valid": true, + "per_stream_tps": [ + 42.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08092900900010136, + "decode_tps": 42.431546925030815, + "prefill_tps": null, + "itl_mean_ms": 23.56737080000469, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.01750100162462, + "round_gap_p50_ms": 53.20598799880827, + "round_gap_p95_ms": 71.36478800020996, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.56737080000469, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.01750100162462, + "round_gap_p50_ms": 53.20598799880827, + "round_gap_p95_ms": 71.36478800020996, + "round_gap_mean_ms": 55.645181055566624, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 29188.794335409, + "first": 29188.875264418, + "last": 29194.884943972, + "end": 29194.885266349, + "elapsed_s": 6.090930940001272, + "chunk_offsets_s": [ + 0.08092900900010136, + 0.1389008680016559, + 0.19799201400019228, + 0.2540542830029153, + 0.3058097130015085, + 0.35437394300242886, + 0.4054119590000482, + 0.45352753800034407, + 0.5058085100026801, + 0.5678860400003032, + 0.6297907680018398, + 0.6758272010010842, + 0.7357346170028904, + 0.7946402580018912, + 0.8505535620024602, + 0.9046530930027075, + 0.9527347220027877, + 0.9976045550029085, + 1.0414225960012118, + 1.0898085510016244, + 1.138917730000685, + 1.1894454380017123, + 1.2460278400030802, + 1.2960349480017612, + 1.3475559540020186, + 1.4026741670022602, + 1.46086818600088, + 1.5207523450008011, + 1.5888774590021058, + 1.6417828049998207, + 1.6938324580005428, + 1.7418918630028202, + 1.7933043650009495, + 1.8530205370007025, + 1.915715277002164, + 1.9737884900023346, + 2.0326824200019473, + 2.1040472080021573, + 2.1729885970016767, + 2.2206093900022097, + 2.2721297580028477, + 2.32091039900115, + 2.365218326001923, + 2.4141104300033476, + 2.466623074000381, + 2.5218574940008693, + 2.5760807120022946, + 2.6379626720008673, + 2.6978643350012135, + 2.7477044470033434, + 2.795846673001506, + 2.8569355420004285, + 2.930925089000084, + 2.994233745001111, + 3.0474874540013843, + 3.099506439000834, + 3.15422683900033, + 3.211038155001006, + 3.268506750002416, + 3.320611696002743, + 3.365343097000732, + 3.4174711460000253, + 3.4676255020021927, + 3.5141718400009267, + 3.573421038003289, + 3.6282920660014497, + 3.6724373350007227, + 3.71951984000043, + 3.7676397240029473, + 3.830654545003199, + 3.890441127001395, + 3.935416766002163, + 3.980323865001992, + 4.0294565410004, + 4.088226935000421, + 4.170981888000824, + 4.252734273002716, + 4.329808136000793, + 4.395500234000792, + 4.442808823001542, + 4.488860223002121, + 4.534836460003135, + 4.59618420300103, + 4.667545857002551, + 4.7316340080033115, + 4.7846844350024185, + 4.835180696001771, + 4.899557082000683, + 4.964941603000625, + 5.026955454002746, + 5.086779843000841, + 5.133973033000075, + 5.183506409000984, + 5.2314804080015165, + 5.277374347002478, + 5.324398318000021, + 5.372236533999967, + 5.433142958001554, + 5.501887017002446, + 5.551927861000877, + 5.602985401001206, + 5.6507483269997465, + 5.709369169002457, + 5.789079489000869, + 5.857154091001576, + 5.9231715920032, + 5.990622815003007, + 6.0374025750024884, + 6.090608563001297 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 53.22984849954082, + "p95": 71.36478800020996, + "max": 82.75495300040348 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.54927533713357, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.076826109998365, + "end_to_end_tps": 42.12725448549471, + "timing_valid": true, + "per_stream_tps": [ + 42.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08370574400032638, + "decode_tps": 42.54927533713357, + "prefill_tps": null, + "itl_mean_ms": 23.502162894118218, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.29809700098122, + "round_gap_p50_ms": 53.50077800176223, + "round_gap_p95_ms": 71.37194800088764, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.502162894118218, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.29809700098122, + "round_gap_p50_ms": 53.50077800176223, + "round_gap_p95_ms": 71.37194800088764, + "round_gap_mean_ms": 55.491217944445786, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 29194.885989947, + "first": 29194.969695691, + "last": 29200.962747229, + "end": 29200.962816057, + "elapsed_s": 6.076826109998365, + "chunk_offsets_s": [ + 0.08370574400032638, + 0.14224728100089123, + 0.20052139800100122, + 0.2566502390000096, + 0.3087478880006529, + 0.3567851139996492, + 0.4079548599984264, + 0.45468148899817606, + 0.5066921039979206, + 0.5669322760004434, + 0.6276596299976518, + 0.6736318160001247, + 0.7374311649982701, + 0.7987302389992692, + 0.8505708019984013, + 0.8986049729974184, + 0.9457503029989311, + 0.9911170679988572, + 1.0354480159985542, + 1.0828385159984464, + 1.1318166210003255, + 1.1817720990002272, + 1.2361315189991728, + 1.285518876000424, + 1.336535433998506, + 1.3918196550002904, + 1.4500301259977277, + 1.5116386230001808, + 1.5807406589992752, + 1.6336015089982538, + 1.6844739970001683, + 1.7367782920009631, + 1.7911702060009702, + 1.8457061419976526, + 1.9059970729977067, + 1.9635878679982852, + 2.0207152379989566, + 2.0920343559992034, + 2.1613199450002867, + 2.2087511089994223, + 2.260078685998451, + 2.309187562997977, + 2.3536811289995967, + 2.4033019239977875, + 2.4568027019995498, + 2.5122314000000188, + 2.5674765589974413, + 2.6279314040002646, + 2.686723031998554, + 2.740880613997433, + 2.7907732149978983, + 2.8503385029980564, + 2.921710450998944, + 2.9824521059999825, + 3.0358482519986865, + 3.087786235999374, + 3.1415424389997497, + 3.2004763539989654, + 3.256920830997842, + 3.3076530849975825, + 3.353365279999707, + 3.4060345129983034, + 3.4552000049989147, + 3.50154270299754, + 3.558327588998509, + 3.612607854000089, + 3.657938730000751, + 3.7059886770002777, + 3.758076923997578, + 3.8199686799998744, + 3.8774201800006267, + 3.9233075800002553, + 3.9699925009990693, + 4.018690781998885, + 4.076841665999382, + 4.159922446000564, + 4.241951452000649, + 4.319706489000964, + 4.385037366999313, + 4.433163443998637, + 4.479207286000019, + 4.524263284998597, + 4.585296034998464, + 4.6582125929999165, + 4.724510690000898, + 4.7824154299996735, + 4.8309334500008845, + 4.890465926997422, + 4.955828720998397, + 5.017291834999924, + 5.0776705680000305, + 5.123895455999445, + 5.173182586000621, + 5.222214964000159, + 5.267700409000099, + 5.314714239997556, + 5.362331537999125, + 5.423305738000636, + 5.489982177998172, + 5.540968835000967, + 5.591359385998658, + 5.639722189000167, + 5.69837294899844, + 5.781851662999543, + 5.848461515000963, + 5.910801019999781, + 5.97743259499839, + 6.025156367999443, + 6.076757282000472 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 53.62849050106888, + "p95": 71.37194800088764, + "max": 83.47871400110307 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.48390711723287, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.0854872279996925, + "end_to_end_tps": 42.067297228417246, + "timing_valid": true, + "per_stream_tps": [ + 42.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08291847900181892, + "decode_tps": 42.48390711723287, + "prefill_tps": null, + "itl_mean_ms": 23.538324694113815, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.87540500186151, + "round_gap_p50_ms": 53.53931699937675, + "round_gap_p95_ms": 71.99782400130061, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.538324694113815, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.87540500186151, + "round_gap_p50_ms": 53.53931699937675, + "round_gap_p95_ms": 71.99782400130061, + "round_gap_mean_ms": 55.576599972213174, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 29200.963556812, + "first": 29201.046475291, + "last": 29207.048748088, + "end": 29207.04904404, + "elapsed_s": 6.0854872279996925, + "chunk_offsets_s": [ + 0.08291847900181892, + 0.14066561499930685, + 0.19961592500112602, + 0.2556826500003808, + 0.3086716889993113, + 0.3565024790004827, + 0.40629817499939236, + 0.45382658400194487, + 0.5062140590016497, + 0.5684675939992303, + 0.6305361000013363, + 0.6800381659995764, + 0.7451891900018381, + 0.8037394610000774, + 0.8546735769996303, + 0.9041707169999427, + 0.9514041030015505, + 0.9970608459989307, + 1.041231068000343, + 1.0902560760005144, + 1.1384274709998863, + 1.1892117980023613, + 1.2454834820018732, + 1.2947336610013735, + 1.3458771670011629, + 1.401670159000787, + 1.459261157000583, + 1.5193584549997468, + 1.5886443780000263, + 1.642183694999403, + 1.6975887100015825, + 1.7471185360009258, + 1.7985963110004377, + 1.8538369750021957, + 1.9156631469995773, + 1.9731956620016717, + 2.0300190029993246, + 2.102016827000625, + 2.169698165002046, + 2.218951847000426, + 2.2694224340011715, + 2.316997829999309, + 2.3613654260007024, + 2.410309782000695, + 2.463182167000923, + 2.5183580510019965, + 2.5730587620018923, + 2.634345205002319, + 2.698367265998968, + 2.751967689000594, + 2.799324728999636, + 2.856566759000998, + 2.927746510998986, + 2.9890270960022463, + 3.0418232310003077, + 3.0954219779996492, + 3.148231664999912, + 3.2071874200009916, + 3.264887950001139, + 3.3165024890004133, + 3.361758426999586, + 3.4145884970021143, + 3.4646208680023847, + 3.51169807900078, + 3.5701407279993873, + 3.624125548001757, + 3.6742945530022553, + 3.7250626429995464, + 3.7733047799993074, + 3.831284632000461, + 3.8877525009993406, + 3.934350163999625, + 3.9791956520020904, + 4.027858944002219, + 4.086839073999727, + 4.168432881000626, + 4.249839529999008, + 4.326793258001999, + 4.393144580000808, + 4.440222024000832, + 4.486384175001149, + 4.531862555999396, + 4.592972551999992, + 4.667014473001473, + 4.7366158709992305, + 4.791629596002167, + 4.836200874000497, + 4.896290004999173, + 4.961334068000724, + 5.0215297380018455, + 5.083649702999537, + 5.129746728001919, + 5.177945990999433, + 5.225887204000173, + 5.271719218999351, + 5.318357669999386, + 5.36601912200058, + 5.427057225999306, + 5.495116070000222, + 5.546234870998887, + 5.598465899001894, + 5.647803117000876, + 5.712414067002101, + 5.794115372998931, + 5.856779876001383, + 5.919271443999605, + 5.985146849001467, + 6.032437959001982, + 6.085191276000842 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 53.56903199935914, + "p95": 71.99782400130061, + "max": 81.70130599683034 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 65.4, + "per_stream_tps": 33.4, + "ttft_s_max": 0.16, + "request": { + "ttft_s": 0.121, + "decode_tps": 33.376, + "prefill_tps": null, + "itl_mean_ms": 29.964, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.525, + "round_gap_p50_ms": 69.43, + "round_gap_p95_ms": 84.938, + "tokens_per_chunk": 2.349 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 29.97, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.426, + "round_gap_p50_ms": 69.32, + "round_gap_p95_ms": 85.781, + "tokens_per_chunk": 2.349 + }, + "timing_valid": true, + "end_to_end_tps": 64.96350926791062, + "makespan_s": 7.881347632999677, + "steady_tps": 67.5, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 65.48208307266705, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 7.869032721999247, + "end_to_end_tps": 65.0651761262366, + "timing_valid": true, + "steady_tps": 67.54922863538233, + "per_stream_tps": [ + 33.1, + 33.7 + ], + "ttft_s": [ + 0.16, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16042114700030652, + "decode_tps": 33.08153886520611, + "prefill_tps": null, + "itl_mean_ms": 30.22833986274325, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.98532900132705, + "round_gap_p50_ms": 69.09175300097559, + "round_gap_p95_ms": 84.9237570000696, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08034566200149129, + "decode_tps": 33.74434230298318, + "prefill_tps": null, + "itl_mean_ms": 29.634597439215604, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.14752899948508, + "round_gap_p50_ms": 69.5180349975999, + "round_gap_p95_ms": 86.13328099818318, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 29.93146865097943, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.98532900132705, + "round_gap_p50_ms": 69.31950400030473, + "round_gap_p95_ms": 86.13328099818318, + "round_gap_mean_ms": 70.67152320370143, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 29207.050159133, + "first": 29207.21058028, + "last": 29214.918806945, + "end": 29214.91910402, + "elapsed_s": 7.868944886999088, + "chunk_offsets_s": [ + 0.16042114700030652, + 0.22036889900118695, + 0.30029496700080927, + 0.38316503999885754, + 0.4666707270007464, + 0.5368850220002059, + 0.6019065370019234, + 0.6643130859993107, + 0.7274187180009903, + 0.7944981879991246, + 0.8674346750012774, + 0.9268006829988735, + 0.9962748229991121, + 1.0613485020003282, + 1.1235206999990623, + 1.182839413999318, + 1.2376385550014675, + 1.2924363679994713, + 1.3474399940023432, + 1.403240971001651, + 1.4622740840022743, + 1.5248339739991934, + 1.5969247450011608, + 1.6658334900021146, + 1.730300849001651, + 1.8000139249998028, + 1.8723402329997043, + 1.9373664170016127, + 2.014354793001985, + 2.0949868079987937, + 2.1623054670017154, + 2.2281804869999178, + 2.29170729100224, + 2.3613530799993896, + 2.4407038320023275, + 2.525505378998787, + 2.6037188440022874, + 2.6810488350020023, + 2.758405471002334, + 2.8206813639990287, + 2.8817372519988567, + 2.956772205001471, + 3.0407239939995634, + 3.127023842000199, + 3.203706472999329, + 3.2784236980005517, + 3.3452424200004316, + 3.422153614999843, + 3.5008850560006977, + 3.5882548350018624, + 3.673178592001932, + 3.7492937770002754, + 3.831323247999535, + 3.9052937039996323, + 3.980377578001935, + 4.061323230002017, + 4.129782162999618, + 4.207027611999365, + 4.274315930000739, + 4.3323028619997785, + 4.39074453899957, + 4.463613865002117, + 4.526686453002185, + 4.592309571999067, + 4.671966455000074, + 4.7537758110011055, + 4.811250982998899, + 4.874859383002331, + 4.95232804799889, + 5.034313377000217, + 5.108351102000597, + 5.170731656002317, + 5.234075907999795, + 5.297913592999976, + 5.367005346000951, + 5.446798764001869, + 5.536370099001942, + 5.634454146002099, + 5.732685915001639, + 5.8088980370011996, + 5.872171856000932, + 5.928782676000992, + 5.992116043998976, + 6.064941304000968, + 6.127662204999069, + 6.204946599002142, + 6.264508880001813, + 6.322713528999884, + 6.384144842999376, + 6.458031173002382, + 6.526058817002195, + 6.595212265001464, + 6.657933454000158, + 6.716997806001018, + 6.77641249899898, + 6.837916458000109, + 6.901328087998991, + 6.980029161000857, + 7.054024480999942, + 7.125560123000469, + 7.196407924002415, + 7.274021542001719, + 7.351558923000994, + 7.413212474999455, + 7.474816173002182, + 7.552069263001613, + 7.636860311999044, + 7.7022632750013145, + 7.767709767998895, + 7.816287707999436, + 7.8686478119998355 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 69.12260050012264, + "p95": 84.9237570000696, + "max": 98.23176899953978 + } + }, + { + "start": 29207.050071298, + "first": 29207.13041696, + "last": 29214.687239307, + "end": 29214.687376548, + "elapsed_s": 7.6373052500021, + "chunk_offsets_s": [ + 0.08034566200149129, + 0.22029187800217187, + 0.3000141340016853, + 0.3830427259999851, + 0.46645870599968475, + 0.5372669860007591, + 0.6017293780023465, + 0.6641355749998183, + 0.7280303330007882, + 0.7942608400007884, + 0.8672723950003274, + 0.9267800920024456, + 0.9960554570025124, + 1.0611706070012588, + 1.1243159229998128, + 1.1827249560010387, + 1.2376096120024158, + 1.2922835270001087, + 1.3472053660007077, + 1.403132639999967, + 1.461902775001363, + 1.524837845001457, + 1.5967141319997609, + 1.6656352280006104, + 1.7302740020022611, + 1.8000635860007606, + 1.8721936000001733, + 1.9372162479994586, + 2.014129459999822, + 2.095009510001546, + 2.162922907002212, + 2.2284981030024937, + 2.291841876001854, + 2.361359910999454, + 2.440460180001537, + 2.5254836349995458, + 2.6035752190000494, + 2.6812067780010693, + 2.75834926600146, + 2.8208184280010755, + 2.8817406430025585, + 2.956554567001149, + 3.04073204100132, + 3.126865321999503, + 3.2037357350018283, + 3.278377252001519, + 3.3451354009994247, + 3.4220848660006595, + 3.500679642002069, + 3.588032846000715, + 3.6733454470013385, + 3.7493199029995594, + 3.8314674319990445, + 3.9053262779998477, + 3.9800891910017526, + 4.0614529830018, + 4.130202542000916, + 4.207062649002182, + 4.274702374001208, + 4.33191894400079, + 4.391121143999044, + 4.463602569001523, + 4.526640950000001, + 4.592157227001735, + 4.672451807000471, + 4.753631096999015, + 4.811261925999133, + 4.874806009000167, + 4.952716875999613, + 5.034287121001398, + 5.108308543000021, + 5.170709591999184, + 5.23427143400113, + 5.2978202369995415, + 5.366868968001654, + 5.446953348000534, + 5.536594279001292, + 5.634308456999861, + 5.732707465002022, + 5.808740843000123, + 5.8722135970019735, + 5.9287805470012245, + 5.992318738000904, + 6.065674355999363, + 6.127854867001588, + 6.204718609002157, + 6.26455177400203, + 6.322664907002036, + 6.384140602000116, + 6.458168141001806, + 6.526195578000625, + 6.595515082000929, + 6.658334105999529, + 6.716846613002417, + 6.776275080999767, + 6.837682500001392, + 6.901080723000632, + 6.980367672000284, + 7.054257765001239, + 7.12535766700239, + 7.196720308002114, + 7.274167661998945, + 7.350854522999725, + 7.413101918999018, + 7.4747563660021115, + 7.5519545640017895, + 7.637168009001471 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 69.65380949804967, + "p95": 86.13328099818318, + "max": 139.94621600068058 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 65.38145399853643, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 7.881347632999677, + "end_to_end_tps": 64.96350926791062, + "timing_valid": true, + "steady_tps": 67.4801350858703, + "per_stream_tps": [ + 33.0, + 33.7 + ], + "ttft_s": [ + 0.16, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.1614464369995403, + "decode_tps": 33.0324227083121, + "prefill_tps": null, + "itl_mean_ms": 30.27328660783835, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.8061690008326, + "round_gap_p50_ms": 68.81218999842531, + "round_gap_p95_ms": 84.9530199993751, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08090323500073282, + "decode_tps": 33.707849497619065, + "prefill_tps": null, + "itl_mean_ms": 29.666680458824118, + "itl_p50_ms": 0.0, + "itl_p95_ms": 83.18626400068752, + "round_gap_p50_ms": 69.65062200106331, + "round_gap_p95_ms": 85.78137600125046, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 29.969983533331234, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.99253100267379, + "round_gap_p50_ms": 69.12952899801894, + "round_gap_p95_ms": 85.78137600125046, + "round_gap_mean_ms": 70.76246112036542, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 29214.921011662, + "first": 29215.082458099, + "last": 29222.802146184, + "end": 29222.802213429, + "elapsed_s": 7.881201766998856, + "chunk_offsets_s": [ + 0.1614464369995403, + 0.22068939999735449, + 0.3007302419973712, + 0.3844470609983546, + 0.4682489339975291, + 0.5383159199991496, + 0.5974829659971874, + 0.6553624219995982, + 0.7216390039975522, + 0.7927935739971872, + 0.8661316789984994, + 0.9277153889997862, + 0.9965275789982115, + 1.0622452499992505, + 1.1249406069982797, + 1.1841572060002363, + 1.2380548709988943, + 1.2927497019991279, + 1.3478706569985661, + 1.4041043639990676, + 1.4625633159994322, + 1.5253321409982163, + 1.593137191997812, + 1.6611736249979003, + 1.7289624689983611, + 1.802357160999236, + 1.8754504659991653, + 1.9411020029983774, + 2.0183257899989258, + 2.099604184997588, + 2.1695092969966936, + 2.2354718440001307, + 2.2978558909999265, + 2.3669854199979454, + 2.4455933069984894, + 2.5305463269978645, + 2.604416706999473, + 2.677087602998654, + 2.7598937719994865, + 2.825657079996745, + 2.886121416999231, + 2.9622631199999887, + 3.0463660360001086, + 3.133279056997708, + 3.2099825919976865, + 3.282565325996984, + 3.347548879999522, + 3.4240638949995628, + 3.501856967999629, + 3.5862518879985146, + 3.6686528119971626, + 3.751728251998429, + 3.8357992339988414, + 3.9104139319970272, + 3.9849791359993105, + 4.066960958996788, + 4.134864648996881, + 4.213915985998028, + 4.281362526999146, + 4.341423521997058, + 4.399698974997591, + 4.472674724998797, + 4.536169303999486, + 4.597209040999587, + 4.672290095997596, + 4.759425078998902, + 4.818892881998181, + 4.88209641299909, + 4.96047514899692, + 5.040411132998997, + 5.114902492998226, + 5.177926018997823, + 5.242940262996854, + 5.307178634997399, + 5.374558142997557, + 5.454095438999502, + 5.540017065999564, + 5.633356099999219, + 5.732982347999496, + 5.8121418679984345, + 5.87603406999915, + 5.933494649998465, + 5.996973083998455, + 6.069722951997392, + 6.134074214998691, + 6.2110278499967535, + 6.26974738699937, + 6.327797191999707, + 6.389220891996956, + 6.465346144999785, + 6.532805569997436, + 6.599396677996992, + 6.657726919998822, + 6.7191294049989665, + 6.78344954899876, + 6.844483520999347, + 6.908983351997449, + 6.986759488998359, + 7.061454323997168, + 7.133747445997869, + 7.205898060998152, + 7.284203280996735, + 7.3593934399978025, + 7.423405113997433, + 7.486450041997159, + 7.564348829997471, + 7.645643562998885, + 7.7091922469990095, + 7.780330822999531, + 7.829359649997059, + 7.8811345219983195 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 68.97085949822213, + "p95": 84.9530199993751, + "max": 99.62624800027697 + } + }, + { + "start": 29214.920865796, + "first": 29215.001769031, + "last": 29222.566772548, + "end": 29222.566911695, + "elapsed_s": 7.646045899000455, + "chunk_offsets_s": [ + 0.08090323500073282, + 0.22064783499808982, + 0.3009247609988961, + 0.38496334299998125, + 0.46859899799892446, + 0.5382496199999878, + 0.5975265809975099, + 0.6552716109981702, + 0.7216804749987205, + 0.7926872429998184, + 0.8661331830007839, + 0.9275669319977169, + 0.996604183997988, + 1.0620519949989102, + 1.1253249740002502, + 1.1844941669987747, + 1.2384617339994293, + 1.292782933000126, + 1.3483249739983876, + 1.4041901199998392, + 1.4626229940004123, + 1.5257295960000192, + 1.5933760299994901, + 1.661619430000428, + 1.7298729569993156, + 1.8026753070007544, + 1.8754465950005397, + 1.941115618999902, + 2.0182641930005047, + 2.099529787999927, + 2.1694286450001528, + 2.2355442410007527, + 2.2978074769998784, + 2.3667695260010078, + 2.4454592170004616, + 2.530873032999807, + 2.6044248350008274, + 2.6771062909974717, + 2.760292554998159, + 2.8254489610008022, + 2.8862033660007, + 2.962358924, + 3.046114494998619, + 3.1333412139974826, + 3.2104261249987758, + 3.282420948999061, + 3.3472890359989833, + 3.423978842998622, + 3.50195918800091, + 3.58584708899798, + 3.6688396200006537, + 3.7515925139996398, + 3.836011657000199, + 3.9106131560001813, + 3.9856349010005943, + 4.066980685998715, + 4.134811787000217, + 4.214108873999066, + 4.28146298700085, + 4.341613466000126, + 4.400128188997769, + 4.472784896999656, + 4.536268772000767, + 4.597252879997541, + 4.672498022999207, + 4.759523171000183, + 4.819164263000857, + 4.882205177000287, + 4.960663124998973, + 5.0404999460006366, + 5.115102788000513, + 5.177998864000983, + 5.243160414000158, + 5.307251287998952, + 5.374755015000119, + 5.45417274799911, + 5.53995412400036, + 5.633764275000431, + 5.733197587000177, + 5.812528700000257, + 5.8766947309995885, + 5.933872394998616, + 5.996974747999047, + 6.07001391599988, + 6.133924364999984, + 6.211207537999144, + 6.270069038000656, + 6.328051388998574, + 6.389096480997978, + 6.4653790070005925, + 6.533002633997967, + 6.599457540000003, + 6.657813956000609, + 6.719328260998736, + 6.783654500999546, + 6.844477921000362, + 6.909214879000501, + 6.986966409000161, + 7.061866290998296, + 7.133842225997796, + 7.206003048999264, + 7.284293276999961, + 7.3592759239982115, + 7.42332270100087, + 7.486638962000143, + 7.564411098999699, + 7.645906752000883 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 69.7747395006445, + "p95": 85.78137600125046, + "max": 139.744599997357 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 65.27977517018266, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 7.894844841997838, + "end_to_end_tps": 64.85244615275242, + "timing_valid": true, + "steady_tps": 66.75700523570087, + "per_stream_tps": [ + 33.0, + 33.7 + ], + "ttft_s": [ + 0.16, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16145063099975232, + "decode_tps": 32.976334621143955, + "prefill_tps": null, + "itl_mean_ms": 30.324777192150826, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.24423600040609, + "round_gap_p50_ms": 69.34249700134387, + "round_gap_p95_ms": 83.87665600093897, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08197786699747667, + "decode_tps": 33.67054584822908, + "prefill_tps": null, + "itl_mean_ms": 29.69954821960795, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.99115800036816, + "round_gap_p50_ms": 69.79000600040308, + "round_gap_p95_ms": 84.29625700227916, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 30.01216270587939, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.42634799898951, + "round_gap_p50_ms": 69.51356599893188, + "round_gap_p95_ms": 84.10152799842763, + "round_gap_mean_ms": 70.86205083332634, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 29222.803847046, + "first": 29222.965297677, + "last": 29230.698115861, + "end": 29230.698455929, + "elapsed_s": 7.894608882998, + "chunk_offsets_s": [ + 0.16145063099975232, + 0.22087807899879408, + 0.30138144299780834, + 0.3852580989987473, + 0.46935962699717493, + 0.5393187619993114, + 0.5984160469997732, + 0.6572945560001244, + 0.7204968489968451, + 0.7875685759972839, + 0.864177210998605, + 0.9298970219970215, + 1.000185226999747, + 1.0663511129969265, + 1.1275378609971085, + 1.1851070519987843, + 1.2403184439972392, + 1.2953035259997705, + 1.3491441070000292, + 1.4058628019993193, + 1.4658050059988454, + 1.5286672419970273, + 1.5965579900002922, + 1.663408600998082, + 1.7260246549994918, + 1.7955382209984236, + 1.8715242910002416, + 1.937941462998424, + 2.0136828109971248, + 2.095746284998313, + 2.165088781999657, + 2.2310020639997674, + 2.292427690998011, + 2.3591518699977314, + 2.4405813369994576, + 2.5236028239996813, + 2.5972097369995026, + 2.6708189210003184, + 2.7483537499974773, + 2.8107195499978843, + 2.877140556000086, + 2.9567138370002795, + 3.0396852329977264, + 3.1263893069990445, + 3.2036835529979726, + 3.27606636499695, + 3.3427078209970205, + 3.4179586069985817, + 3.4964584409972304, + 3.579866444997606, + 3.6604624100000365, + 3.7365164779985207, + 3.8191933409980265, + 3.897913246997632, + 3.975256450998131, + 4.057500686998537, + 4.125730797997676, + 4.204391904997465, + 4.272382136998203, + 4.331932690998656, + 4.3921762760001, + 4.46336960699773, + 4.527362031996745, + 4.5895924090000335, + 4.6649003099992115, + 4.747962616998848, + 4.804221258997131, + 4.8741005579977355, + 4.957832184998551, + 5.03964988199732, + 5.113743646998046, + 5.1773654030002945, + 5.242763302998355, + 5.30567620399961, + 5.3756432679983845, + 5.455520473999059, + 5.542946833997121, + 5.635990655999194, + 5.7313587319986254, + 5.807648545996926, + 5.876694155998848, + 5.937573497998528, + 6.0027954959987255, + 6.076437477997388, + 6.139269409999542, + 6.216258186999767, + 6.276790054998855, + 6.334473095997964, + 6.397453821999079, + 6.472446540999954, + 6.540389013000095, + 6.606006109999726, + 6.664457251998101, + 6.724104686996725, + 6.784360341996944, + 6.8480201799975475, + 6.917670430997532, + 6.998095127997658, + 7.072267853000085, + 7.144402935999096, + 7.21522044899757, + 7.295066801998473, + 7.371340382000199, + 7.434481664997293, + 7.497150467999745, + 7.574675588999526, + 7.655277627000032, + 7.719398268000077, + 7.785974552996777, + 7.836662843998056, + 7.894268814998213 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 69.42803150013788, + "p95": 83.87665600093897, + "max": 95.36807599943131 + } + }, + { + "start": 29222.803611087, + "first": 29222.885588954, + "last": 29230.45897375, + "end": 29230.459148256, + "elapsed_s": 7.655537168997398, + "chunk_offsets_s": [ + 0.08197786699747667, + 0.22129655899698264, + 0.30174296499899356, + 0.3853684949972376, + 0.46966475199951674, + 0.5396188619997702, + 0.5985061229985149, + 0.6575732009987405, + 0.7206549069996981, + 0.7879115379982977, + 0.8645336929985206, + 0.9302340329995786, + 1.0001619639988348, + 1.0666263499988418, + 1.1276862879967666, + 1.185262053997576, + 1.2404163449973566, + 1.2956782799992652, + 1.3492134959997202, + 1.4065100089974294, + 1.4664049989987689, + 1.5286540749984852, + 1.5965715249985806, + 1.663440983997134, + 1.7261138359972392, + 1.795582138998725, + 1.8720165279992216, + 1.938506561997201, + 2.013384933998168, + 2.0957560449969606, + 2.1649626589969557, + 2.2309719049990235, + 2.2924250349969952, + 2.35916169399934, + 2.440316306998284, + 2.5235529219971795, + 2.5971633059998567, + 2.6707968579976296, + 2.7483094959970913, + 2.8111151509983756, + 2.877133373000106, + 2.956749178996688, + 3.0401964769989718, + 3.126192657997308, + 3.203695648997382, + 3.2764352949998283, + 3.3426492149992555, + 3.4187379059985687, + 3.49694259899843, + 3.5804089849989396, + 3.6605439899976773, + 3.737054105997231, + 3.819280360999983, + 3.898005371000181, + 3.9749707329974626, + 4.05820798899731, + 4.125804138999229, + 4.204768114999752, + 4.272346585999912, + 4.33200327999657, + 4.39207189499939, + 4.463925057996676, + 4.527438668999821, + 4.5894378059965675, + 4.664964226998563, + 4.7479553849989315, + 4.804679033997672, + 4.8744690399980755, + 4.957775833998312, + 5.0402021819973015, + 5.1137320149973675, + 5.177256077997299, + 5.243162455997663, + 5.3061338029983744, + 5.375372348997189, + 5.455546936998871, + 5.54280920599922, + 5.635904003000178, + 5.73131231699881, + 5.807678912999108, + 5.876799431996915, + 5.937743122998654, + 6.002546304000134, + 6.076496435998706, + 6.139435178996791, + 6.216020193998702, + 6.2768886589983595, + 6.334501077999448, + 6.397330193998641, + 6.472188533996814, + 6.540778997998132, + 6.606080090998148, + 6.664378165998642, + 6.724265688997548, + 6.784656521998841, + 6.84813327899974, + 6.91807532799794, + 6.998620420999941, + 7.072645870997803, + 7.144491747996653, + 7.215664352999738, + 7.29510033599945, + 7.371351867997873, + 7.434510559000046, + 7.497152643998561, + 7.574763360997167, + 7.655362662997504 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 69.85896849982964, + "p95": 84.29625700227916, + "max": 139.31869199950597 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 94.6, + "per_stream_tps": 24.6, + "ttft_s_max": 0.36, + "request": { + "ttft_s": 0.219, + "decode_tps": 24.658, + "prefill_tps": null, + "itl_mean_ms": 40.562, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.974, + "round_gap_p50_ms": 94.378, + "round_gap_p95_ms": 107.474, + "tokens_per_chunk": 2.339 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 40.148, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.969, + "round_gap_p50_ms": 94.597, + "round_gap_p95_ms": 107.754, + "tokens_per_chunk": 2.359 + }, + "timing_valid": true, + "end_to_end_tps": 94.26462204889411, + "makespan_s": 10.86303618200327, + "steady_tps": 99.1, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 94.85868079749046, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.841400176999741, + "end_to_end_tps": 94.45274441325739, + "timing_valid": true, + "steady_tps": 99.1471764362119, + "per_stream_tps": [ + 24.3, + 25.0, + 23.7, + 26.9 + ], + "ttft_s": [ + 0.25, + 0.17, + 0.09, + 0.34 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.25081672099986463, + "decode_tps": 24.321901276758275, + "prefill_tps": null, + "itl_mean_ms": 41.115206768624965, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.96886399757932, + "round_gap_p50_ms": 93.72184600215405, + "round_gap_p95_ms": 107.24199099786347, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16933099000016227, + "decode_tps": 25.006421033565744, + "prefill_tps": null, + "itl_mean_ms": 39.989728984316265, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.07074900285807, + "round_gap_p50_ms": 93.82342700337176, + "round_gap_p95_ms": 107.40967500169063, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.088415480000549, + "decode_tps": 23.714670199372616, + "prefill_tps": null, + "itl_mean_ms": 42.16799101960336, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.1628179991676, + "round_gap_p50_ms": 93.23777400277322, + "round_gap_p95_ms": 107.2426960017765, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3354811429999245, + "decode_tps": 26.876103644602434, + "prefill_tps": null, + "itl_mean_ms": 37.20777435686186, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.97914799978025, + "round_gap_p50_ms": 93.60119399934774, + "round_gap_p95_ms": 107.22751200228231, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 40.12017528235161, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.97914799978025, + "round_gap_p50_ms": 93.60102099890355, + "round_gap_p95_ms": 107.2426960017765, + "round_gap_mean_ms": 95.16878787906661, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 29230.700540833, + "first": 29230.951357554, + "last": 29241.43573528, + "end": 29241.435822254, + "elapsed_s": 10.735281420998945, + "chunk_offsets_s": [ + 0.25081672099986463, + 0.42147641599876806, + 0.5253319759976876, + 0.6332262289979553, + 0.7329573530005291, + 0.8224103479988116, + 0.904003019000811, + 1.0071615300003032, + 1.1100200639994, + 1.1985690709989285, + 1.2857460929990339, + 1.3702099090005504, + 1.4579676789981022, + 1.5456030530003773, + 1.6393656129985175, + 1.7400024700000358, + 1.8382691349979723, + 1.9417907269998977, + 2.041001215999131, + 2.122566748999816, + 2.222205865000433, + 2.313331950997963, + 2.3983200319999014, + 2.497511628000211, + 2.5864430670008005, + 2.6784546589988167, + 2.775560531998053, + 2.8593155659982585, + 2.9633782479977526, + 3.0706137209999724, + 3.1657111540007463, + 3.2527446339991, + 3.342876572998648, + 3.430016608999722, + 3.524028453997744, + 3.6275320519998786, + 3.7247710550000193, + 3.819877791000181, + 3.9202388519988745, + 4.01381991100061, + 4.103767239997978, + 4.200013343997853, + 4.301415042998997, + 4.404390051000519, + 4.497991071999422, + 4.5893949329984025, + 4.670942246000777, + 4.763313691000803, + 4.8633135780000885, + 4.969547474000137, + 5.073516337997717, + 5.16529697399892, + 5.265393175999634, + 5.3696059200010495, + 5.476847910998913, + 5.592482452000695, + 5.677222452999558, + 5.7712028969981475, + 5.869360792999942, + 5.966856389000895, + 6.073924034000811, + 6.174505700000736, + 6.270368862999021, + 6.373483269999269, + 6.472611179000523, + 6.572293598001124, + 6.659281837997696, + 6.748317485998996, + 6.845257525998022, + 6.945788094999443, + 7.049030706999474, + 7.138271282998176, + 7.235646855999221, + 7.3295980539987795, + 7.4175661090012, + 7.514379986998392, + 7.615804250999645, + 7.725064465001196, + 7.84442569499879, + 7.940132903000631, + 8.03259529999923, + 8.116792908000207, + 8.203761348999251, + 8.293508618000487, + 8.373094709000725, + 8.463031859999319, + 8.555396584997652, + 8.6351667179988, + 8.715070782000112, + 8.802901406001183, + 8.890359511999122, + 8.982447718000913, + 9.068374192000192, + 9.159018471000309, + 9.245700732000842, + 9.33439842000007, + 9.433558467000694, + 9.539035235000483, + 9.629230206999637, + 9.722453151000082, + 9.824176432997774, + 9.913795133001258, + 10.004444619000424, + 10.089532737998525, + 10.175909040997794, + 10.269630886999948, + 10.36603622199982, + 10.435817928999313, + 10.508814949000225, + 10.59362243600117, + 10.665554286999395, + 10.73519444699923 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 93.72184600215405, + "p95": 107.24199099786347, + "max": 170.65969499890343 + } + }, + { + "start": 29230.700339368, + "first": 29230.869670358, + "last": 29241.067051249, + "end": 29241.067128878, + "elapsed_s": 10.366789510000672, + "chunk_offsets_s": [ + 0.16933099000016227, + 0.42140029100119136, + 0.5258761170007347, + 0.633535066001059, + 0.7330780530028278, + 0.8227768950018799, + 0.9048368940020737, + 1.007157370000641, + 1.1106887610003469, + 1.1993028380020405, + 1.286200092999934, + 1.3702614270005142, + 1.45793484800015, + 1.54611484400084, + 1.63946458600185, + 1.7404236719994515, + 1.8380661020019033, + 1.941761176, + 2.041141723002511, + 2.1226988560010795, + 2.2228639550012304, + 2.313977945002989, + 2.3989589709999564, + 2.497944541002653, + 2.5870042800015653, + 2.678914308002277, + 2.7759111119994486, + 2.859448905001045, + 2.963657198000874, + 3.0710668730025645, + 3.166337452999869, + 3.2536842670015176, + 3.342809791000036, + 3.429933876002906, + 3.5241378259997873, + 3.628078257999732, + 3.7256204190016433, + 3.8203147530002752, + 3.9203384810025454, + 4.013760840000032, + 4.1041234680014895, + 4.200364980002632, + 4.301440946001094, + 4.405058333002671, + 4.4978354449995095, + 4.589777880002657, + 4.671321544999955, + 4.7636904300015885, + 4.863680382000894, + 4.969911604999652, + 5.07398235400251, + 5.16579534199991, + 5.265961109002092, + 5.369719995000196, + 5.476814487999945, + 5.592994227001327, + 5.677708180999616, + 5.771146115002921, + 5.869895607000217, + 5.966779783000675, + 6.073868818999472, + 6.175048434000928, + 6.271020265001425, + 6.373416120000911, + 6.472548636000283, + 6.572737471000437, + 6.659686000002694, + 6.74869812900215, + 6.8451154499998665, + 6.946112355999503, + 7.049303209001664, + 7.138584920001449, + 7.236247076001746, + 7.3305671910020465, + 7.4178571550000925, + 7.514240934000554, + 7.616243324002426, + 7.725518274000933, + 7.844938943002489, + 7.940458973000204, + 8.032648593001795, + 8.117222446002415, + 8.20357671400052, + 8.293800352999824, + 8.373651011002949, + 8.4630834410018, + 8.55547280600149, + 8.635701037001127, + 8.715425730999414, + 8.80318523599999, + 8.890825656999368, + 8.982899223999993, + 9.068420029001572, + 9.159478296001907, + 9.246201996000309, + 9.334780936002062, + 9.434260941001412, + 9.539064993001375, + 9.629347818001406, + 9.722553547002462, + 9.824038244001713, + 9.913934088002861, + 10.004395100000693, + 10.089445395002258, + 10.176335923999432, + 10.270159351002803, + 10.36671188100081 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 94.01368850012659, + "p95": 107.40967500169063, + "max": 252.0693010010291 + } + }, + { + "start": 29230.700162942, + "first": 29230.788578422, + "last": 29241.541416132, + "end": 29241.54148229, + "elapsed_s": 10.841319347997342, + "chunk_offsets_s": [ + 0.088415480000549, + 0.4212830469987239, + 0.5254128369997488, + 0.633493979999912, + 0.7335547879993101, + 0.8230736689984042, + 0.904779119999148, + 1.0074533919978421, + 1.1105894199972681, + 1.1992877829979989, + 1.2858227779979643, + 1.3702418920001946, + 1.458566458997666, + 1.5458117979978852, + 1.639234016998671, + 1.7401056659982714, + 1.8380481339991093, + 1.9422109519982769, + 2.041096443997958, + 2.1226031469996087, + 2.222495455000171, + 2.314022455997474, + 2.398859982000431, + 2.4977682909993746, + 2.587062229998992, + 2.6786779959984415, + 2.775539332000335, + 2.859416521998355, + 2.963673070000368, + 3.070714596997277, + 3.165898842999013, + 3.253728040999704, + 3.3428067339991685, + 3.4299417629990785, + 3.5246314729993173, + 3.6277422519997344, + 3.7249777350007207, + 3.819964539998182, + 3.9207751539979654, + 4.014012928000739, + 4.104247095998289, + 4.200279206997948, + 4.301453120999213, + 4.40492038499724, + 4.497776950000116, + 4.589872931999707, + 4.671088928000245, + 4.7634898599972075, + 4.863461732998985, + 4.969696460000705, + 5.074099101999309, + 5.165398937999271, + 5.265948885000398, + 5.3696768439986045, + 5.476919540000381, + 5.592738890998589, + 5.677713156997925, + 5.77109038799972, + 5.869335928997316, + 5.967037854999944, + 6.073754086999543, + 6.174691692998749, + 6.2706302929982485, + 6.374024003998784, + 6.472772499997518, + 6.573070867998467, + 6.659485189997213, + 6.74848066299819, + 6.845343891000084, + 6.946491335998871, + 7.049528913998074, + 7.13901551500021, + 7.235933628999192, + 7.3303967639985785, + 7.41810097099733, + 7.514326242999232, + 7.616375015997619, + 7.725342198998987, + 7.845163814999978, + 7.9404207009974925, + 8.032901193000725, + 8.117008963999979, + 8.203696021999349, + 8.293648964998283, + 8.373244462000002, + 8.463038129997585, + 8.555562066998391, + 8.635293831997842, + 8.715298822000477, + 8.80307399099911, + 8.890874086999247, + 8.982739435999974, + 9.068665492999571, + 9.159331659997406, + 9.245575662000192, + 9.334543389999453, + 9.434117807999428, + 9.538956547999987, + 9.62928066799941, + 9.722499659997993, + 9.824499588998151, + 9.914028020997648, + 10.004014390997327, + 10.089338358000532, + 10.176063434999378, + 10.270400015000632, + 10.366606395000417, + 10.435776537000493, + 10.509010541998578, + 10.593734062000294, + 10.666007264000655, + 10.735297994000575, + 10.78815770700021, + 10.841253189999406 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 93.23777400277322, + "p95": 107.2426960017765, + "max": 332.8675669981749 + } + }, + { + "start": 29230.700082113, + "first": 29231.035563256, + "last": 29240.523545717, + "end": 29240.523852332, + "elapsed_s": 9.823770219001744, + "chunk_offsets_s": [ + 0.3354811429999245, + 0.42203766100283246, + 0.5262214169997605, + 0.633896478000679, + 0.7327239520018338, + 0.8228220130004047, + 0.9046802430020762, + 1.007313828002225, + 1.110562669000501, + 1.1991003330003878, + 1.2860009630021523, + 1.3707311550024315, + 1.45854785100164, + 1.546125099001074, + 1.639459625002928, + 1.7405282120016636, + 1.8386576490011066, + 1.9421910650016798, + 2.041261686001235, + 2.122848322000209, + 2.2227771249999932, + 2.313863389001199, + 2.3988507659996685, + 2.4980438810016494, + 2.5869829680013936, + 2.678992705001292, + 2.7758824080010527, + 2.8599074660014594, + 2.9638866140012396, + 3.070685812999727, + 3.166912234002666, + 3.253454738001892, + 3.3435921010022867, + 3.430618061000132, + 3.524155344002793, + 3.6280458260007435, + 3.7255393490013375, + 3.820275025002047, + 3.9207641140019405, + 4.014222808000341, + 4.104134315002739, + 4.200536494001426, + 4.3017880380029965, + 4.40475872600291, + 4.498358435001137, + 4.589779494999675, + 4.67132863200095, + 4.763711725001485, + 4.863696589000028, + 4.969903173001512, + 5.073894660003134, + 5.165671441001905, + 5.265908646000753, + 5.37011892600276, + 5.477362341000116, + 5.593003122001392, + 5.677742595002201, + 5.771752382002887, + 5.8696763170009945, + 5.967378626999562, + 6.074606139001844, + 6.174897094002517, + 6.270903356002236, + 6.37401592400056, + 6.47314909600027, + 6.5731131870015815, + 6.659835195001506, + 6.748717600003147, + 6.845792420001089, + 6.946443464999902, + 7.049754777999624, + 7.13864187800209, + 7.2363349920015025, + 7.32993618600085, + 7.417889537002338, + 7.514912288002961, + 7.616177501000493, + 7.7256015820021275, + 7.844813040999725, + 7.940643781999825, + 8.033105906000856, + 8.117388214999664, + 8.20409213700259, + 8.293891213001189, + 8.373486535001575, + 8.463771659000486, + 8.55579641100121, + 8.635844342999917, + 8.715455233003013, + 8.803291904001526, + 8.89088642200295, + 8.983282123001118, + 9.06893991600009, + 9.159299692000786, + 9.246218986001622, + 9.334790326000075, + 9.433972404000087, + 9.539736348000588, + 9.629516164000961, + 9.722714437000832, + 9.823463603999699 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 93.80549050001719, + "p95": 107.22751200228231, + "max": 119.21145899759722 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 94.63632025849839, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.86303618200327, + "end_to_end_tps": 94.26462204889411, + "timing_valid": true, + "steady_tps": 100.11859922746777, + "per_stream_tps": [ + 23.9, + 25.0, + 24.3, + 26.6 + ], + "ttft_s": [ + 0.08, + 0.19, + 0.36, + 0.27 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08433012400200823, + "decode_tps": 23.889756613565833, + "prefill_tps": null, + "itl_mean_ms": 41.85894465882288, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.39191300226958, + "round_gap_p50_ms": 94.46567000122741, + "round_gap_p95_ms": 107.8708370005188, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1879150060012762, + "decode_tps": 24.994514843451523, + "prefill_tps": null, + "itl_mean_ms": 40.00877817646445, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.61175300204195, + "round_gap_p50_ms": 95.56413600148517, + "round_gap_p95_ms": 107.96761699748458, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3575964830015437, + "decode_tps": 24.274720791779394, + "prefill_tps": null, + "itl_mean_ms": 41.19511851764115, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.66766600054689, + "round_gap_p50_ms": 94.37672100102645, + "round_gap_p95_ms": 107.53897199901985, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2727136080029595, + "decode_tps": 26.646196307695092, + "prefill_tps": null, + "itl_mean_ms": 37.52880855686004, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.61328500221134, + "round_gap_p50_ms": 95.40755999842077, + "round_gap_p95_ms": 107.7544729996589, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 40.14791247744713, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.39191300226958, + "round_gap_p50_ms": 95.0481010004296, + "round_gap_p95_ms": 107.7544729996589, + "round_gap_mean_ms": 95.23458308603738, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 29241.543797229, + "first": 29241.628127353, + "last": 29252.302158241, + "end": 29252.302277261, + "elapsed_s": 10.75848003200008, + "chunk_offsets_s": [ + 0.08433012400200823, + 0.44182757000089623, + 0.5476245220015699, + 0.6553303300024709, + 0.7539877950002847, + 0.8444161560000794, + 0.9252008960029343, + 1.0215960270033975, + 1.1263882720013498, + 1.2177057590015465, + 1.3062357360031456, + 1.389472258000751, + 1.4768016540001554, + 1.5653825820008933, + 1.6598482520021207, + 1.7600710510014324, + 1.8588819870019506, + 1.9617009190005774, + 2.0560003700011293, + 2.143905477001681, + 2.2480696530001296, + 2.3418343040029868, + 2.4269911360024707, + 2.5272681140013447, + 2.616011772002821, + 2.708347408002737, + 2.8069304170021496, + 2.8891866770027264, + 2.9883370620009373, + 3.090166235000652, + 3.1920442700029525, + 3.279780930002744, + 3.3712495980034873, + 3.458635373001016, + 3.5544386860019586, + 3.6583791140001267, + 3.755697169002815, + 3.852326523003285, + 3.952327271003014, + 4.039801555001759, + 4.132223700002214, + 4.232496871001786, + 4.332854310003313, + 4.435481513002742, + 4.528688483002043, + 4.619039180000982, + 4.7038307360016915, + 4.795916553001007, + 4.8970766450001975, + 4.9991024890005065, + 5.097555104002822, + 5.194836936003412, + 5.296527607002645, + 5.399683644001925, + 5.509831872001087, + 5.623765661002835, + 5.708579995000036, + 5.805242548001843, + 5.9016938420027145, + 5.995601734000957, + 6.098775947000831, + 6.206682173000445, + 6.301745988002949, + 6.404867690001993, + 6.506461082000897, + 6.607596854002622, + 6.693104332000075, + 6.781451989001653, + 6.877663143000973, + 6.976684509001643, + 7.073276805000205, + 7.169283502000326, + 7.267041101000359, + 7.36079233500277, + 7.44826957800251, + 7.545339746000536, + 7.648805582000932, + 7.756676419001451, + 7.874314929002139, + 7.968158720002975, + 8.054611367002508, + 8.143271378001373, + 8.235908034002932, + 8.326021422002668, + 8.406759174002218, + 8.497106125003484, + 8.588879295002698, + 8.667747385003167, + 8.744322448001185, + 8.83043039200129, + 8.918510045001312, + 9.006548242003191, + 9.085223025002051, + 9.18171638900094, + 9.26880173900281, + 9.35699123700033, + 9.455471008001041, + 9.55886292100331, + 9.64871954300179, + 9.74203860200214, + 9.84197491100349, + 9.932500843002344, + 10.017136643000413, + 10.101506882001559, + 10.196995132002485, + 10.293924381003308, + 10.391879777002032, + 10.460462851002376, + 10.535230307003076, + 10.618459527002415, + 10.689530552001088, + 10.758361012001842 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 94.46567000122741, + "p95": 107.8708370005188, + "max": 357.497445998888 + } + }, + { + "start": 29241.54471645, + "first": 29241.732631456, + "last": 29251.934869891, + "end": 29251.935287304, + "elapsed_s": 10.390570854000543, + "chunk_offsets_s": [ + 0.1879150060012762, + 0.4410504250008671, + 0.5471391280007083, + 0.6545792799988703, + 0.7537989199990989, + 0.8441380039985233, + 0.9240010749999783, + 1.0206422150004073, + 1.1256106629989517, + 1.2166757740014873, + 1.3051055610012554, + 1.3893140389991459, + 1.4764755200012587, + 1.5642496069995104, + 1.65929770799994, + 1.7592165960013517, + 1.8581173220009077, + 1.9611250160014606, + 2.0552075449995755, + 2.1427576710011635, + 2.2474127120003686, + 2.340870347998134, + 2.426998464998178, + 2.5274133259990776, + 2.6153102249991207, + 2.707868557998154, + 2.805892511998536, + 2.888410796000244, + 2.988155947998166, + 3.089597580001282, + 3.191650185999606, + 3.2790869349992136, + 3.3702874340015114, + 3.457754646999092, + 3.5533187830005772, + 3.6573639919988636, + 3.7550509329994384, + 3.8516174889991817, + 3.9520136489991273, + 4.039383856001223, + 4.131533399999171, + 4.231267931001639, + 4.332667484999547, + 4.434759742998722, + 4.527903305999644, + 4.618665032001445, + 4.702596324001206, + 4.7958782839996275, + 4.896327899001335, + 4.997922762999224, + 5.096935426998243, + 5.193974898000306, + 5.296394924000197, + 5.398830900998291, + 5.508953273998486, + 5.623273323999456, + 5.708142520998081, + 5.804088724999019, + 5.900984887000959, + 5.994301066999469, + 6.097912820001511, + 6.205880436998996, + 6.301236035000329, + 6.404142654999305, + 6.506014295999194, + 6.606956937001087, + 6.692547916998592, + 6.7809084379987326, + 6.876890989999083, + 6.976097487000516, + 7.072680119999859, + 7.168427846998384, + 7.266399792999437, + 7.3607604179997, + 7.447403507001582, + 7.544991070000833, + 7.6481427539984, + 7.756117812001321, + 7.872824604000925, + 7.96695478699985, + 8.053928091998387, + 8.142593733999092, + 8.234675908999634, + 8.325441919998411, + 8.40554912800144, + 8.495989677998296, + 8.588151397001639, + 8.6665187159997, + 8.74371469000107, + 8.82990584900108, + 8.91701570299847, + 9.004683446000854, + 9.084133617001498, + 9.181231443999422, + 9.268432999000652, + 9.356151613999828, + 9.454061815999012, + 9.558379288999276, + 9.64812050600085, + 9.74138086200037, + 9.840908476999175, + 9.931655860000319, + 10.016503718001331, + 10.100390002000495, + 10.196314767999866, + 10.293304319999152, + 10.390153440999711 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 95.65593150000495, + "p95": 107.96761699748458, + "max": 253.1354189995909 + } + }, + { + "start": 29241.543878571, + "first": 29241.901475054, + "last": 29252.406230276, + "end": 29252.406389872, + "elapsed_s": 10.862511301002087, + "chunk_offsets_s": [ + 0.3575964830015437, + 0.44182801799979643, + 0.5476972559990827, + 0.6555098120006733, + 0.7543349840016162, + 0.8449175330024445, + 0.9247283170006995, + 1.0213649770012125, + 1.126520284000435, + 1.2179552400011744, + 1.3059947660003672, + 1.3896076940000057, + 1.4769557130020985, + 1.5654508200022974, + 1.6598514190009155, + 1.7602019110017864, + 1.8591724899997644, + 1.9617114790016785, + 2.0564923230012937, + 2.144065632001002, + 2.248528279000311, + 2.3416657480011054, + 2.427449730999797, + 2.527710100999684, + 2.6164157439998235, + 2.708098199000233, + 2.8063135060001514, + 2.8891609810016234, + 2.9893767120011034, + 3.0899476490012603, + 3.1920425899988913, + 3.279841296000086, + 3.3713347150005575, + 3.458438722002029, + 3.554682711001078, + 3.658423977001803, + 3.75555688500026, + 3.8526014429990028, + 3.9525493920009467, + 4.039943247000338, + 4.132024073001958, + 4.232602963998943, + 4.332961890999286, + 4.4361761020009, + 4.529455932999554, + 4.619785799000965, + 4.703106659999321, + 4.796403835000092, + 4.897473306002212, + 4.998397308001586, + 5.097603711001284, + 5.194229848999385, + 5.296607012001914, + 5.3995164959997055, + 5.51029683500019, + 5.62433361299918, + 5.708386960999633, + 5.804655988002196, + 5.9021858439991775, + 5.995255902998906, + 6.099378138002066, + 6.206917110001086, + 6.3019024950008315, + 6.404661135002243, + 6.506775936999475, + 6.60793762799949, + 6.692950815999211, + 6.781883768999251, + 6.877833315000316, + 6.97687522400156, + 7.073457360002067, + 7.168896072002099, + 7.267085436000343, + 7.36146215700137, + 7.448093422000966, + 7.545432191000145, + 7.64910549400156, + 7.756816462999268, + 7.874051064001833, + 7.967847193001944, + 8.054657493001287, + 8.143014025001321, + 8.235432334000507, + 8.325839650002308, + 8.406950447999407, + 8.49730616799934, + 8.58959773999959, + 8.667869413002336, + 8.74414882000201, + 8.830809245999262, + 8.91797183500239, + 9.005729160002375, + 9.0851892339997, + 9.182107114000246, + 9.26894205500139, + 9.35715289700238, + 9.455197286999464, + 9.558864953000011, + 9.648296481998841, + 9.742525341000146, + 9.84197631799907, + 9.932257569002104, + 10.017602839001484, + 10.101031517999218, + 10.196859119001601, + 10.293350380001357, + 10.391189985999517, + 10.46046133100026, + 10.535029544000281, + 10.618633074001991, + 10.689356332000898, + 10.75880763999885, + 10.811049203999573, + 10.862351705000037 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 94.37672100102645, + "p95": 107.53897199901985, + "max": 117.23460100256489 + } + }, + { + "start": 29241.54335369, + "first": 29241.816067298, + "last": 29251.38591348, + "end": 29251.386078803, + "elapsed_s": 9.842725113001507, + "chunk_offsets_s": [ + 0.2727136080029595, + 0.4421733520030102, + 0.5480004150012974, + 0.6555426120030461, + 0.7546658550018037, + 0.8451563110029383, + 0.9257838870034902, + 1.0215632280014688, + 1.1269284160007373, + 1.2184062990017992, + 1.3064255380013492, + 1.390398631003336, + 1.4780283700019936, + 1.5655141460010782, + 1.6605625510019308, + 1.760682137002732, + 1.8599780350014044, + 1.962398147003114, + 2.056885992002208, + 2.1444503890015767, + 2.248979722000513, + 2.342035946003307, + 2.4277330030017765, + 2.5285685880007804, + 2.6171483470025123, + 2.7089958530013973, + 2.8070474050000485, + 2.88952785100264, + 2.9890329300033045, + 3.0907251310018182, + 3.1927591290004784, + 3.2802673960031825, + 3.371765999003401, + 3.4591653890020098, + 3.555289478001214, + 3.658902763003425, + 3.7563428150024265, + 3.8532935680013907, + 3.9529182610021962, + 4.0403378430019075, + 4.133236327001214, + 4.2327454240003135, + 4.333770588000334, + 4.436659752002015, + 4.529739285000687, + 4.619726248001825, + 4.7034801700028765, + 4.7964542180016, + 4.897571863002668, + 4.999802469003043, + 5.097763498000859, + 5.194983907000278, + 5.297171236001304, + 5.400260171001719, + 5.510551804003626, + 5.624345036001614, + 5.709209513002861, + 5.804976714000077, + 5.903013101000397, + 5.99551084000268, + 6.099557700003061, + 6.207546644000104, + 6.302301764000731, + 6.405110850002529, + 6.507166517003498, + 6.608515196003282, + 6.693804872000328, + 6.782062291000329, + 6.8785714540026675, + 6.977261197000189, + 7.074232907001715, + 7.169640467000136, + 7.267994338002609, + 7.3617505640031595, + 7.448504514002707, + 7.5461304760028725, + 7.649427388001641, + 7.7571818610013, + 7.874943215003441, + 7.968804062002164, + 8.054823344002216, + 8.14360693600247, + 8.236259623001388, + 8.32654191900292, + 8.407742618001066, + 8.49745947500196, + 8.589871555002901, + 8.668714046001696, + 8.744963677003398, + 8.831025351002609, + 8.918695799002307, + 9.00676229900273, + 9.08560512600161, + 9.182396082000196, + 9.270060664002813, + 9.35790127600194, + 9.455508813000051, + 9.559464136000315, + 9.64931199000057, + 9.743261193001672, + 9.84255979000227 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 95.58738049781823, + "p95": 107.7544729996589, + "max": 169.4597440000507 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 94.59034399803278, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.869747101001849, + "end_to_end_tps": 94.20642361638933, + "timing_valid": true, + "steady_tps": 98.9960771263841, + "per_stream_tps": [ + 24.1, + 25.1, + 24.2, + 26.1 + ], + "ttft_s": [ + 0.17, + 0.25, + 0.35, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16671856100219884, + "decode_tps": 24.062872328896958, + "prefill_tps": null, + "itl_mean_ms": 41.55779851764023, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.96926300018094, + "round_gap_p50_ms": 94.3786440002441, + "round_gap_p95_ms": 107.3584910009231, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.24984247100292123, + "decode_tps": 25.133808020124373, + "prefill_tps": null, + "itl_mean_ms": 39.78704696078328, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.42301199873327, + "round_gap_p50_ms": 94.94288600035361, + "round_gap_p95_ms": 108.0270019992895, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3459973370008811, + "decode_tps": 24.233673422917608, + "prefill_tps": null, + "itl_mean_ms": 41.264895443144304, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.03055299684638, + "round_gap_p50_ms": 94.13592599958065, + "round_gap_p95_ms": 107.15047099802177, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08607448900147574, + "decode_tps": 26.10858307708347, + "prefill_tps": null, + "itl_mean_ms": 38.30158063528692, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.16881000128342, + "round_gap_p50_ms": 94.9627629997849, + "round_gap_p95_ms": 107.9252589988755, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 40.227830389213686, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.96926300018094, + "round_gap_p50_ms": 94.59702299864148, + "round_gap_p95_ms": 107.80854299810017, + "round_gap_mean_ms": 95.424155806972, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 29252.409292676, + "first": 29252.576011237, + "last": 29263.173249859, + "end": 29263.173429951, + "elapsed_s": 10.764137275000394, + "chunk_offsets_s": [ + 0.16671856100219884, + 0.43275675200129626, + 0.5396972529997583, + 0.6470557440006814, + 0.7450926670026092, + 0.8350389030019869, + 0.9154213390029327, + 1.01197738799965, + 1.1127683680024347, + 1.202038078001351, + 1.2972959050021018, + 1.3834292650026327, + 1.4707945990012377, + 1.5580336470011389, + 1.6513112130014633, + 1.752549008000642, + 1.8498210960024153, + 1.9527896140025405, + 2.0471038420000696, + 2.1294505990008474, + 2.2296896030020434, + 2.329450971999904, + 2.416434067999944, + 2.516130055002577, + 2.6043931470012467, + 2.696876357000292, + 2.797245303001546, + 2.8810770670024795, + 2.981643448001705, + 3.0825942830015265, + 3.180033959000866, + 3.2709204250022594, + 3.3658692830031214, + 3.451943249001488, + 3.5459462510007143, + 3.649915514000895, + 3.746044254003209, + 3.8426976690025185, + 3.9423514250011067, + 4.029501463002816, + 4.117613059999712, + 4.214620275000925, + 4.3217551710004045, + 4.425410548999935, + 4.517849291001767, + 4.610221410999657, + 4.693561948002753, + 4.786193102001562, + 4.886387827002181, + 4.988241007002216, + 5.085697657003038, + 5.178398329000629, + 5.284019800001261, + 5.3913541750007425, + 5.501563011999679, + 5.615793815002689, + 5.703234221000457, + 5.800509361000877, + 5.898418822001986, + 5.9911936890021025, + 6.094448378000379, + 6.195662269001332, + 6.29688475200237, + 6.403451412999857, + 6.507246783003211, + 6.608161808002478, + 6.6958180040019215, + 6.78346861600221, + 6.880653460000758, + 6.98024942900156, + 7.076448536001408, + 7.167544633000944, + 7.267745643002854, + 7.367427369001234, + 7.454005016003066, + 7.548774883001897, + 7.650782782002352, + 7.758959124999819, + 7.875989333002508, + 7.970367977002752, + 8.056087134002155, + 8.139266450001742, + 8.224702622002951, + 8.31916000000274, + 8.40058005100218, + 8.492824797001958, + 8.58580890200028, + 8.666175485999702, + 8.746344219001912, + 8.832140496000648, + 8.921254790002422, + 9.008625766000478, + 9.088461562001612, + 9.17855925600088, + 9.268195921002189, + 9.362548889999744, + 9.46219712400125, + 9.567203809001512, + 9.658820813001512, + 9.751674188002653, + 9.853104830002849, + 9.944630524001695, + 10.0287249230023, + 10.110586111000885, + 10.19954060300006, + 10.298411690000648, + 10.395383274000778, + 10.466383823000797, + 10.538961710000876, + 10.624121732002095, + 10.696521511003084, + 10.763957183000457 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 94.3786440002441, + "p95": 107.3584910009231, + "max": 266.0381909990974 + } + }, + { + "start": 29252.408999596, + "first": 29252.658842067, + "last": 29262.804539042, + "end": 29262.80470947, + "elapsed_s": 10.395709874002932, + "chunk_offsets_s": [ + 0.24984247100292123, + 0.4332622430010815, + 0.5393110390032234, + 0.6474607250020199, + 0.7451880240005266, + 0.8348374360029993, + 0.9160260429998743, + 1.0114004590031982, + 1.1128054870023334, + 1.2016500080026162, + 1.297254626002541, + 1.3833423870019033, + 1.4705891470002825, + 1.5578434119997837, + 1.6515134000001126, + 1.7521053070013295, + 1.8498881180021272, + 1.9536462960022618, + 2.046707563000382, + 2.1295757640000375, + 2.230267396000272, + 2.3294432110014895, + 2.416232712999772, + 2.5166463460009254, + 2.6045839420003176, + 2.6969587700004922, + 2.797089722000237, + 2.8816106530030083, + 2.9821525540028233, + 3.0831988750032906, + 3.180074757001421, + 3.2715061210001295, + 3.3662758490027045, + 3.4520364460004203, + 3.5459944250033004, + 3.6504174370020337, + 3.745661062999716, + 3.8428206740027235, + 3.942740231002972, + 4.029911916000856, + 4.117829006001557, + 4.214850316002412, + 4.321978780000791, + 4.425486882002588, + 4.518203730000096, + 4.610567466002976, + 4.694038687001012, + 4.7864230790000875, + 4.886800136002421, + 4.98847312900034, + 5.085633946000598, + 5.178926059001242, + 5.284151396001107, + 5.391600200000539, + 5.501789310001186, + 5.615922707002028, + 5.703467271003319, + 5.800711196003249, + 5.898880106000433, + 5.991357252001762, + 6.094680643000174, + 6.195715211000788, + 6.297433090003324, + 6.403813226999773, + 6.507732625999779, + 6.608176286001253, + 6.696688990003167, + 6.7838163190026535, + 6.8806992980025825, + 6.980463294999936, + 7.076520886002982, + 7.167957838002621, + 7.268161184001656, + 7.367642707002233, + 7.454392990002816, + 7.5489900130014576, + 7.651128437002626, + 7.7591554390019155, + 7.8761968790022365, + 7.97113976500259, + 8.056135580001865, + 8.138990215000376, + 8.22481866599992, + 8.319673971000157, + 8.401680103001127, + 8.492807517002802, + 8.586689280000428, + 8.666404632000194, + 8.746908332002931, + 8.832331322002574, + 8.921658107003168, + 9.00883924700247, + 9.088677700001426, + 9.178794129002199, + 9.268197840003268, + 9.363484498000616, + 9.462729446000594, + 9.56738537199999, + 9.659264833000634, + 9.752184814002248, + 9.853252282002359, + 9.945053249000921, + 10.028932213001099, + 10.11082309700214, + 10.199957024000469, + 10.299159559002874, + 10.395539446002658 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 95.09325599901786, + "p95": 108.0270019992895, + "max": 183.4197719981603 + } + }, + { + "start": 29252.409697834, + "first": 29252.755695171, + "last": 29263.278243509, + "end": 29263.27857443, + "elapsed_s": 10.868876596003247, + "chunk_offsets_s": [ + 0.3459973370008811, + 0.4327548000001116, + 0.5388915140028985, + 0.6467000570009986, + 0.7449638380021497, + 0.8344131110025046, + 0.9150820670001849, + 1.0113285240004188, + 1.112582581001334, + 1.201549690002139, + 1.296826461002638, + 1.382830656002625, + 1.4701109520028695, + 1.557282658002805, + 1.6507076119996782, + 1.7519260790031694, + 1.849355075002677, + 1.9525980350008467, + 2.0466584770001646, + 2.1294963420004933, + 2.2296317489999637, + 2.3288182520009286, + 2.4157333340008336, + 2.515011155002867, + 2.604024260999722, + 2.696352866001689, + 2.7966863560031925, + 2.880485850000696, + 2.9810115760010376, + 3.082070055999793, + 3.1789996170009545, + 3.2706438550012535, + 3.3650955270022678, + 3.451003321002645, + 3.5451392470022256, + 3.649287242002174, + 3.7453564000024926, + 3.8420347420033067, + 3.941664514000877, + 4.029353402002016, + 4.117531350002537, + 4.214016721001826, + 4.321161761003168, + 4.4251923140000144, + 4.517934761002834, + 4.610351104001893, + 4.6936554340027214, + 4.78606697700161, + 4.886471377001726, + 4.988168513002165, + 5.085563852000632, + 5.178486951001105, + 5.283901627000887, + 5.390785933002917, + 5.501391097001033, + 5.615479647000029, + 5.703275756000949, + 5.800290679002501, + 5.898103198000172, + 5.991169946002628, + 6.094131730002118, + 6.195317253001122, + 6.296439323003142, + 6.403589794001164, + 6.507156625000789, + 6.6081082570017315, + 6.69557609000185, + 6.78317228800006, + 6.880654292002873, + 6.980083705002471, + 7.076411593003286, + 7.167013109999971, + 7.2675353920021735, + 7.367190767003194, + 7.453933146000054, + 7.548789635002322, + 7.650908442999935, + 7.75886587900095, + 7.875794218001829, + 7.970268635002867, + 8.055885028003104, + 8.138426326000626, + 8.22372931800055, + 8.319217135001963, + 8.401235971003189, + 8.492671089003125, + 8.585690282001451, + 8.66605696100305, + 8.746534054000222, + 8.832051506000425, + 8.92132993999985, + 9.00857493500007, + 9.088261167002202, + 9.178673926002375, + 9.268051029001072, + 9.36238169400167, + 9.462301569001283, + 9.567129331000615, + 9.658027104000212, + 9.751029611001286, + 9.852637354000763, + 9.944131513002503, + 10.028613998001674, + 10.110704653001449, + 10.19963848900079, + 10.298288510002749, + 10.39559229400038, + 10.466436798000359, + 10.538720692002244, + 10.623968679999962, + 10.696668596003292, + 10.764329687001009, + 10.81727323800078, + 10.868545675002679 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 94.13592599958065, + "p95": 107.15047099802177, + "max": 116.92833900087862 + } + }, + { + "start": 29252.408827329, + "first": 29252.494901818, + "last": 29262.26180488, + "end": 29262.26189803, + "elapsed_s": 9.853070700999524, + "chunk_offsets_s": [ + 0.08607448900147574, + 0.4336845990001166, + 0.5395346969999082, + 0.6474599559987837, + 0.7452844850013207, + 0.835078133000934, + 0.916132630998618, + 1.0118688139991718, + 1.1128747490001842, + 1.2021538660010265, + 1.2976193520007655, + 1.38346894300048, + 1.4707126159992185, + 1.5586021119997895, + 1.6516610110011243, + 1.75243417800084, + 1.8499122280009033, + 1.9536191760016663, + 2.0473141229995235, + 2.130057015001512, + 2.230544699999882, + 2.3294666500005405, + 2.416962806000811, + 2.5160848230007105, + 2.604941964000318, + 2.6974250940002094, + 2.7973690740000166, + 2.8814512800017837, + 2.9820364129991503, + 3.0839492550003342, + 3.1796982220002974, + 3.271075059001305, + 3.36603782200109, + 3.45207633300015, + 3.5458600919992023, + 3.6500289020004857, + 3.745984798999416, + 3.8427253949994338, + 3.9423696480007493, + 4.029825404999428, + 4.1182443229990895, + 4.214721503001783, + 4.3219546520012955, + 4.4257740110006125, + 4.518484682001144, + 4.610959344001458, + 4.694331302998762, + 4.786383079001098, + 4.88706195299892, + 4.988849615001527, + 5.086045839001599, + 5.178987545001291, + 5.2845538330002455, + 5.391397676001361, + 5.502136069000699, + 5.616599730001326, + 5.703836877000867, + 5.80078240900184, + 5.89878911499909, + 5.991850423000869, + 6.094688545999816, + 6.195809608001582, + 6.297183079001115, + 6.404206577000878, + 6.507587045001856, + 6.608773519001261, + 6.696565824000572, + 6.783794031001889, + 6.8812001810001675, + 6.9805275479993725, + 7.076996953001071, + 7.16761606900036, + 7.268502613998862, + 7.367999961999885, + 7.4546633179998025, + 7.549389569998311, + 7.651513163000345, + 7.759633491001296, + 7.876084448998881, + 7.970525586999429, + 8.056017774000793, + 8.139964494999731, + 8.224847512999986, + 8.31944415199905, + 8.40139025299868, + 8.49268007900173, + 8.586018480000348, + 8.666292457000964, + 8.746460517999367, + 8.833180581001216, + 8.92135184200015, + 9.008699506000994, + 9.088515686999017, + 9.179080041998532, + 9.268597398000566, + 9.362945310000214, + 9.462580663999688, + 9.567607221000799, + 9.659151971001847, + 9.752428471998428, + 9.85297755099964 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 95.21412449976197, + "p95": 107.9252589988755, + "max": 347.61010999864084 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 103.4, + "per_stream_tps": 21.7, + "ttft_s_max": 0.43, + "request": { + "ttft_s": 0.25, + "decode_tps": 21.731, + "prefill_tps": null, + "itl_mean_ms": 46.017, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.131, + "round_gap_p50_ms": 105.588, + "round_gap_p95_ms": 119.868, + "tokens_per_chunk": 2.286 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 45.359, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.996, + "round_gap_p50_ms": 105.439, + "round_gap_p95_ms": 119.868, + "tokens_per_chunk": 2.315 + }, + "timing_valid": true, + "end_to_end_tps": 102.99935844146873, + "makespan_s": 12.42726187199878, + "steady_tps": 110.2, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 103.29909918144713, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 12.42726187199878, + "end_to_end_tps": 102.99935844146873, + "timing_valid": true, + "steady_tps": 110.15714142471305, + "per_stream_tps": [ + 21.7, + 22.2, + 21.8, + 24.0, + 20.8 + ], + "ttft_s": [ + 0.25, + 0.08, + 0.43, + 0.33, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.24995046900221496, + "decode_tps": 21.697572487829543, + "prefill_tps": null, + "itl_mean_ms": 46.08810504312928, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.6796809991647, + "round_gap_p50_ms": 105.0667199997406, + "round_gap_p95_ms": 118.72439500075416, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08388883600127883, + "decode_tps": 22.203666359931656, + "prefill_tps": null, + "itl_mean_ms": 45.03760702352213, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.6390520009154, + "round_gap_p50_ms": 105.88567599916132, + "round_gap_p95_ms": 119.05426100202021, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.42733277800289216, + "decode_tps": 21.76957678397985, + "prefill_tps": null, + "itl_mean_ms": 45.93566562745015, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.19177299851435, + "round_gap_p50_ms": 103.93805300191161, + "round_gap_p95_ms": 117.82238500018138, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33419838400004664, + "decode_tps": 24.002146697776645, + "prefill_tps": null, + "itl_mean_ms": 41.66294009413048, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.09729400131619, + "round_gap_p50_ms": 106.34641800061218, + "round_gap_p95_ms": 119.13783799900557, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16874120200009202, + "decode_tps": 20.802424754127717, + "prefill_tps": null, + "itl_mean_ms": 48.071319176461635, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.4479690012522, + "round_gap_p50_ms": 103.9317979993939, + "round_gap_p95_ms": 119.21243200049503, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 45.35912739293874, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.29343399888603, + "round_gap_p50_ms": 105.43925299862167, + "round_gap_p95_ms": 118.85683999935281, + "round_gap_mean_ms": 105.53446610583373, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 29263.28016652, + "first": 29263.530116989, + "last": 29275.282583775, + "end": 29275.282713756, + "elapsed_s": 12.00254723600301, + "chunk_offsets_s": [ + 0.24995046900221496, + 0.5209044440016442, + 0.6317799759999616, + 0.746719317001407, + 0.8563553740023053, + 0.956531444000575, + 1.0475733450002735, + 1.149681626000529, + 1.2566485290008131, + 1.3527965180001047, + 1.4572184410026239, + 1.5512994460004847, + 1.6444804970014957, + 1.7400746599996637, + 1.8510144829997444, + 1.9622388900024816, + 2.0757668930018554, + 2.1905452339997282, + 2.2938231860025553, + 2.386447146000137, + 2.5041912110027624, + 2.60709371700068, + 2.697656733002077, + 2.810518291000335, + 2.9141887160003535, + 3.0121738170018943, + 3.120981836000283, + 3.2157284029999573, + 3.323717369003134, + 3.436283140999876, + 3.541349860999617, + 3.6424201230001927, + 3.7399360990020796, + 3.834296291002829, + 3.9352183010014414, + 4.048137013000087, + 4.150888693002344, + 4.253444682002737, + 4.360557238000183, + 4.460919692999596, + 4.563035989001946, + 4.66607334299988, + 4.774424737002846, + 4.882380876002571, + 4.989242720002949, + 5.090527970001858, + 5.179962594000244, + 5.2826231250001, + 5.39502333800192, + 5.51552143400113, + 5.620972685999732, + 5.7218538020024425, + 5.832250783001655, + 5.949317853002867, + 6.066140689003078, + 6.185825828000816, + 6.2852457290027814, + 6.394268677002401, + 6.507005701001617, + 6.607420971002284, + 6.719242209001095, + 6.837966604001849, + 6.945639279001625, + 7.056188673002907, + 7.16489481300232, + 7.275762361001398, + 7.3716733470027975, + 7.484772556999815, + 7.598668994000036, + 7.7106807560012385, + 7.816665181002463, + 7.922472426002059, + 8.0347708219997, + 8.13429651000115, + 8.230049179001071, + 8.338970418000827, + 8.457552449999639, + 8.58194014000037, + 8.716060326001752, + 8.819232604000717, + 8.91308244800166, + 9.004252688002452, + 9.105271478001669, + 9.223675883000396, + 9.318993679000414, + 9.421344984002644, + 9.530117733000225, + 9.619451194001158, + 9.706719804002205, + 9.801869648999855, + 9.909556915001303, + 10.016769359001046, + 10.109875841000758, + 10.21781057300177, + 10.315734916999645, + 10.427627946002758, + 10.53620332400169, + 10.651883005000855, + 10.749036483000964, + 10.850402398002188, + 10.95991235800102, + 11.056129889002477, + 11.151636891001544, + 11.241280772999744, + 11.340759912000067, + 11.451628099002846, + 11.568446388002485, + 11.658170488000906, + 11.747467301000142, + 11.840328838003188, + 11.921424901000137, + 12.002417255000182 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 105.0667199997406, + "p95": 118.72439500075416, + "max": 270.9539749994292 + } + }, + { + "start": 29263.280306565, + "first": 29263.364195401, + "last": 29274.848785192, + "end": 29274.848911765, + "elapsed_s": 11.568605199998274, + "chunk_offsets_s": [ + 0.08388883600127883, + 0.5206279390004056, + 0.6317523289981182, + 0.7473913809990336, + 0.8566910139998072, + 0.9556877200011513, + 1.046874816998752, + 1.1499309639984858, + 1.2563494799978798, + 1.3519808089986327, + 1.45658842299963, + 1.550249502000952, + 1.6442743099978543, + 1.7406756060008775, + 1.85094038799798, + 1.9617972839987488, + 2.0755491370000527, + 2.1907669879983587, + 2.293946222998784, + 2.38652424799875, + 2.504359590999229, + 2.6063015119980264, + 2.6979971729997487, + 2.809676598000806, + 2.9138739709997026, + 3.011798704999819, + 3.121113160999812, + 3.2153642989978835, + 3.323642954997922, + 3.4360794499989424, + 3.5419651259981038, + 3.642809569999372, + 3.7393262250006956, + 3.834303938998346, + 3.934627514998283, + 4.047831003997999, + 4.151276379998308, + 4.253456857000856, + 4.361136759998772, + 4.460945724000339, + 4.561890351000329, + 4.66541359799885, + 4.774003402999369, + 4.88162870099768, + 4.988479040999664, + 5.089684820999537, + 5.179521531001228, + 5.282310348000465, + 5.395506590997684, + 5.514816120998148, + 5.620395498997823, + 5.721245271000953, + 5.831798969000374, + 5.948490224000125, + 6.065906965999602, + 6.1861103179981, + 6.285191841998312, + 6.394402689998969, + 6.507055604000925, + 6.60775987499801, + 6.718749484, + 6.837606323999353, + 6.944856991998677, + 7.055407395000657, + 7.164560484998219, + 7.275097431000177, + 7.371632579997822, + 7.4851520839984005, + 7.59805358399899, + 7.710918382999807, + 7.816344771999866, + 7.922463129998505, + 8.034245088998432, + 8.133820760998788, + 8.230136088997824, + 8.338165075998404, + 8.457219337000424, + 8.581369208997785, + 8.715536160998454, + 8.818841700998746, + 8.912053110998386, + 9.003277045998402, + 9.105090937999194, + 9.222593346999929, + 9.317918790999101, + 9.421068781997747, + 9.529431459999614, + 9.618615147999662, + 9.705528581998806, + 9.80046995199882, + 9.908653590999165, + 10.016439045997686, + 10.110000829998171, + 10.217307816001266, + 10.315835745997902, + 10.427628265999374, + 10.536253082998883, + 10.651606051000272, + 10.748675963001006, + 10.85015536300125, + 10.95867780899789, + 11.055609963997995, + 11.151139989000512, + 11.24086354999963, + 11.340308962000563, + 11.451400440000725, + 11.568478626999422 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 106.00201699890022, + "p95": 119.05426100202021, + "max": 436.73910299912677 + } + }, + { + "start": 29263.280764138, + "first": 29263.708096916, + "last": 29275.421691651, + "end": 29275.421845456, + "elapsed_s": 12.141081318000943, + "chunk_offsets_s": [ + 0.42733277800289216, + 0.52088433300014, + 0.6309826830001839, + 0.7473200070016901, + 0.8564324280014262, + 0.9561507970029197, + 1.0466302950007957, + 1.148997562002478, + 1.2563414960022783, + 1.3518506680011342, + 1.4563399970029423, + 1.54990207100127, + 1.643728931001533, + 1.7393918760026281, + 1.8503241310027079, + 1.9610719260017504, + 2.0754153650013905, + 2.1897525480017066, + 2.293008725002437, + 2.3864982970007986, + 2.5038174590008566, + 2.6062913839996327, + 2.697279158001038, + 2.8097256690016366, + 2.9135071960008645, + 3.0119735340012994, + 3.1202075500004867, + 3.2146952110015263, + 3.323014202000195, + 3.4353852100030053, + 3.5412024880024546, + 3.641710523999791, + 3.7394063680003455, + 3.8339132130022335, + 3.9343026590031513, + 4.047585202002665, + 4.15061458800119, + 4.252777433001029, + 4.360611637002876, + 4.460547622002196, + 4.561835137003072, + 4.6655341559999215, + 4.774267237000458, + 4.88172130699968, + 4.9889115920013865, + 5.089876449001167, + 5.179301776999637, + 5.281981539999833, + 5.394821760000923, + 5.514697468999657, + 5.620151073002489, + 5.721000286001072, + 5.8317670980031835, + 5.948312579999765, + 6.065503232002811, + 6.185170324002684, + 6.284561424999993, + 6.393507094002416, + 6.5065438720012025, + 6.606935382002121, + 6.718743643999915, + 6.837514071001351, + 6.944598471000063, + 7.0550658510001085, + 7.16469555400181, + 7.275324227000965, + 7.371659219999856, + 7.484437701001298, + 7.598556996999832, + 7.710140672999842, + 7.816021484002704, + 7.921763274000114, + 8.034423422002874, + 8.133741819001443, + 8.229071681002097, + 8.338322384999628, + 8.45614476999981, + 8.581172782000067, + 8.715158315000735, + 8.819096368002647, + 8.912616204001097, + 9.003484770000796, + 9.104735427001287, + 9.222468903000845, + 9.317762651000521, + 9.421250346000306, + 9.529314264000277, + 9.618737035001686, + 9.705726003001473, + 9.801425180001388, + 9.90888157000154, + 10.016425303001597, + 10.109060323000449, + 10.217054222001025, + 10.315192529000342, + 10.427277170001616, + 10.535714519002795, + 10.65090629200131, + 10.748719082999742, + 10.849878122000518, + 10.958829262999643, + 11.055278772000747, + 11.150964090000343, + 11.24081027200009, + 11.340498223002214, + 11.45124558000316, + 11.567736836001131, + 11.657341355003155, + 11.746766358002787, + 11.839858065002772, + 11.920539304999693, + 12.002435336002236, + 12.075788623002154, + 12.14092751300268 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 103.93805300191161, + "p95": 117.82238500018138, + "max": 133.9855330006685 + } + }, + { + "start": 29263.280972421, + "first": 29263.615170805, + "last": 29274.239220529, + "end": 29274.23963028, + "elapsed_s": 10.958657859002415, + "chunk_offsets_s": [ + 0.33419838400004664, + 0.5205562290029775, + 0.6316453240033297, + 0.7461792980029713, + 0.8557601400025305, + 0.955324129001383, + 1.0468284980015596, + 1.1491324070011615, + 1.2562225070032582, + 1.3517413110021153, + 1.4558672410021245, + 1.5502637420031533, + 1.6437955050023447, + 1.7397170690019266, + 1.8505288300002576, + 1.9616750320019491, + 2.0747970910015283, + 2.1898943850028445, + 2.2933592130029865, + 2.386084595000284, + 2.503810964000877, + 2.6057547250020434, + 2.6973945960016863, + 2.8092373450017476, + 2.913473612999951, + 3.0112685740023153, + 3.1205563260009512, + 3.2151527050009463, + 3.3234294560024864, + 3.435299741002382, + 3.541646159002994, + 3.642046596, + 3.73906948000149, + 3.8335945320031897, + 3.9343507060002594, + 4.047218731000612, + 4.149921340002038, + 4.252286101000209, + 4.359954180003115, + 4.460440744001971, + 4.562022461002925, + 4.664941594000993, + 4.773476743001083, + 4.881156569001178, + 4.988239911002893, + 5.089573368000856, + 5.179414191003161, + 5.282106353002746, + 5.394803072002105, + 5.513940910001111, + 5.619861896000657, + 5.720651382001961, + 5.831843657000718, + 5.948322836000443, + 6.065707724003005, + 6.185399119000067, + 6.284699278003245, + 6.393220893001853, + 6.506121418002294, + 6.606921703001717, + 6.718003005000355, + 6.837803377002274, + 6.944609415000741, + 7.055137418003142, + 7.164254957002413, + 7.274617763003334, + 7.37122788600027, + 7.484323416003463, + 7.597609563003061, + 7.710498794000159, + 7.816104091001762, + 7.921843705000356, + 8.034298065002076, + 8.133291062000353, + 8.229611397000554, + 8.337673648002237, + 8.456492043002072, + 8.581532422002056, + 8.715340119000757, + 8.818343201000971, + 8.912082616003318, + 9.00312661800126, + 9.104362635000143, + 9.222935533001873, + 9.31782565800313, + 9.420969280999998, + 9.529804350000632, + 9.618398627000715, + 9.705311101002735, + 9.801284127002873, + 9.90809870800149, + 10.015725815002952, + 10.108951286001684, + 10.21710591700321, + 10.31548268300321, + 10.427157285001158, + 10.535628682002425, + 10.650770983000257, + 10.74795990000348, + 10.849842923002143, + 10.958248108003318 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 106.5762279995397, + "p95": 119.13783799900557, + "max": 186.35784500293084 + } + }, + { + "start": 29263.280066634, + "first": 29263.448807836, + "last": 29275.706994226, + "end": 29275.707328506, + "elapsed_s": 12.42726187199878, + "chunk_offsets_s": [ + 0.16874120200009202, + 0.5210714809982164, + 0.6324327129987068, + 0.747880681999959, + 0.8570618539997668, + 0.9561036949999107, + 1.0480297659996722, + 1.1502687639986107, + 1.2573081939990516, + 1.3531826849975914, + 1.4571208269990166, + 1.5509705420008686, + 1.6449326309993921, + 1.740770035998139, + 1.8516167729976587, + 1.9624268379993737, + 2.0755784009998024, + 2.1908718349986884, + 2.2937964829980046, + 2.3870426529974793, + 2.5046608800003014, + 2.6069015779976326, + 2.698169153998606, + 2.810034269998141, + 2.9143365529998846, + 3.012424322998413, + 3.121756746000756, + 3.216157017999649, + 3.324382489998243, + 3.4360133720001613, + 3.541803322998021, + 3.6427893949985446, + 3.739715592997527, + 3.834807015999104, + 3.9356049169982725, + 4.04819913999745, + 4.151131919999898, + 4.253375498999958, + 4.360464088000299, + 4.460818142997596, + 4.562318612999661, + 4.6664381669979775, + 4.774697195000044, + 4.882142817998101, + 4.988906711998425, + 5.0901657059985155, + 5.180113917998824, + 5.282769969999208, + 5.395179543000268, + 5.514649133998319, + 5.621133467000618, + 5.722196962000453, + 5.832140705999336, + 5.949362603998452, + 6.0663543160007976, + 6.186227531998156, + 6.285492348000844, + 6.394414769998548, + 6.506897896000737, + 6.607693124999059, + 6.71916022699952, + 6.838372659000015, + 6.94606520600064, + 7.0558212579999235, + 7.1651076249982, + 7.2762167990003945, + 7.371688610997808, + 7.485301600998355, + 7.599017177999485, + 7.710995901998103, + 7.817081971999869, + 7.922521224998491, + 8.034584985998663, + 8.134127122000791, + 8.229831407999882, + 8.338840292999521, + 8.457086540998716, + 8.581801326999994, + 8.7156367030002, + 8.819568500999594, + 8.912701097000536, + 9.00374205999833, + 9.105180359998485, + 9.223613997000939, + 9.318953871999838, + 9.421229177998612, + 9.530571515999327, + 9.619088753999677, + 9.707074756999646, + 9.801707749000343, + 9.909200971000246, + 10.017045928998414, + 10.110071867999068, + 10.217841612000484, + 10.31577264400039, + 10.428015408997453, + 10.536751167997863, + 10.651356136997492, + 10.748723499000334, + 10.850670264000655, + 10.959036665000895, + 11.056289533000381, + 11.151971907998814, + 11.241782138000417, + 11.340923988998838, + 11.451802543000667, + 11.568493906997901, + 11.658113560999482, + 11.747642865997477, + 11.840370484998857, + 11.921478100000968, + 12.002451173997542, + 12.076573100999667, + 12.14169328799835, + 12.191651642999204, + 12.247811223998724, + 12.311310453998885, + 12.359442128999945, + 12.42692759199781 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 103.93497000040952, + "p95": 119.21243200049503, + "max": 352.3302789981244 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 103.36667885154846, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 12.429713539000659, + "end_to_end_tps": 102.97904259689892, + "timing_valid": true, + "steady_tps": 110.50590032649151, + "per_stream_tps": [ + 21.7, + 22.7, + 21.7, + 23.4, + 20.8 + ], + "ttft_s": [ + 0.26, + 0.34, + 0.42, + 0.09, + 0.18 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.2584104939996905, + "decode_tps": 21.696286275167704, + "prefill_tps": null, + "itl_mean_ms": 46.090837266677354, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.73704799977713, + "round_gap_p50_ms": 105.27614699822152, + "round_gap_p95_ms": 123.17228200117825, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3369091999993543, + "decode_tps": 22.697265956515036, + "prefill_tps": null, + "itl_mean_ms": 44.058169909797414, + "itl_p50_ms": 0.0, + "itl_p95_ms": 117.22231399835437, + "round_gap_p50_ms": 105.58783699889318, + "round_gap_p95_ms": 123.35458600136917, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.41851827099890215, + "decode_tps": 21.73104937447281, + "prefill_tps": null, + "itl_mean_ms": 46.01710588236421, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.24821199802682, + "round_gap_p50_ms": 104.79837700040662, + "round_gap_p95_ms": 121.52452500231448, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.09457860499969684, + "decode_tps": 23.427709426721627, + "prefill_tps": null, + "itl_mean_ms": 42.68449730981386, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.68671000006725, + "round_gap_p50_ms": 105.93224500189535, + "round_gap_p95_ms": 123.86948600033065, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17835993400149164, + "decode_tps": 20.814143607796506, + "prefill_tps": null, + "itl_mean_ms": 48.044253890197176, + "itl_p50_ms": 0.0, + "itl_p95_ms": 117.06977400172036, + "round_gap_p50_ms": 104.275454999879, + "round_gap_p95_ms": 123.12957899848698, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 45.37897285177, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.62059900118038, + "round_gap_p50_ms": 105.27725700012525, + "round_gap_p95_ms": 123.17228200117825, + "round_gap_mean_ms": 105.58063939052327, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 29275.709363518, + "first": 29275.967774012, + "last": 29287.720937515, + "end": 29287.721051784, + "elapsed_s": 12.01168826600042, + "chunk_offsets_s": [ + 0.2584104939996905, + 0.5085205680006766, + 0.6180375330004608, + 0.734774581000238, + 0.8421355290010979, + 0.9423541340001975, + 1.0396146589991986, + 1.1449034350007423, + 1.2515756100001454, + 1.3479025029992044, + 1.446325255001284, + 1.5373330309994344, + 1.6329873780014168, + 1.727222299999994, + 1.8362207290010701, + 1.9493284780000977, + 2.0732723630026157, + 2.18988525000168, + 2.2948064360025455, + 2.3871988960017916, + 2.4986708700016607, + 2.600221135002357, + 2.6910535970018827, + 2.8004707979998784, + 2.9060746890027076, + 3.009974680000596, + 3.1225190200020734, + 3.2172808810028073, + 3.325494520002394, + 3.4340480089995253, + 3.534520612000051, + 3.6353556560025027, + 3.732816580002691, + 3.8283094340004027, + 3.930308757000603, + 4.053481039001781, + 4.158257656999922, + 4.260468011001649, + 4.367543415002729, + 4.459812763001537, + 4.559929162001936, + 4.663053961001424, + 4.770925076001731, + 4.879320388001361, + 4.9910150860014255, + 5.098182730002009, + 5.1875413980014855, + 5.291040936001082, + 5.402299960001983, + 5.51785439699961, + 5.623805835999519, + 5.723864807001519, + 5.833677093000006, + 5.953998182001669, + 6.081298549001076, + 6.20242417800182, + 6.301719623999816, + 6.408548223000253, + 6.514357260999532, + 6.6140844889996515, + 6.725501504999556, + 6.845050143001572, + 6.951493358999869, + 7.07023076599944, + 7.182940165999753, + 7.292374209999252, + 7.389398433002498, + 7.494674580000719, + 7.606275539001217, + 7.7175122990011005, + 7.824178369002766, + 7.929707454000891, + 8.048552791002294, + 8.151872730002651, + 8.248615593001887, + 8.35861544400177, + 8.4691994599998, + 8.590420087002713, + 8.72395422000045, + 8.826761485001043, + 8.920801827000105, + 9.017297350001172, + 9.124070697002026, + 9.240072194999811, + 9.335262667002098, + 9.434007305000705, + 9.537710881002567, + 9.62544830600018, + 9.711566180001682, + 9.806482577001589, + 9.912476265002624, + 10.028212426001119, + 10.125481434002722, + 10.233872092001548, + 10.331401116000052, + 10.43740596600037, + 10.5427759660015, + 10.655451959002676, + 10.753304177000246, + 10.85400459200173, + 10.979067404001398, + 11.079949384002248, + 11.176076861000183, + 11.26343170200198, + 11.361836525000399, + 11.46710620000158, + 11.571065822001401, + 11.659885736000433, + 11.748510695000732, + 11.842999422002322, + 11.923783645001095, + 12.011573997002415 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 105.27614699822152, + "p95": 123.17228200117825, + "max": 250.11007400098606 + } + }, + { + "start": 29275.709230353, + "first": 29276.046139553, + "last": 29287.28097288, + "end": 29287.281078334, + "elapsed_s": 11.571847980998427, + "chunk_offsets_s": [ + 0.3369091999993543, + 0.5084459459976642, + 0.6179143189983733, + 0.7340880519986968, + 0.8418167829986487, + 0.942881529997976, + 1.0393183769992902, + 1.1445956339994154, + 1.2520025289995829, + 1.3478658639978676, + 1.4467007509992982, + 1.5377066229993943, + 1.6329325469996547, + 1.727533253000729, + 1.8363070949999383, + 1.949507913999696, + 2.072862500001065, + 2.1900848139994196, + 2.2950037109985715, + 2.387267823000002, + 2.498713893000968, + 2.600505048998457, + 2.690943918998528, + 2.800877796998975, + 2.9064621520010405, + 3.010062678000395, + 3.122899410998798, + 3.2176197209992097, + 3.3256923080007255, + 3.4344222399995488, + 3.5345610910007963, + 3.635445174000779, + 3.7328202440003224, + 3.828926923997642, + 3.9299647480002022, + 4.053562028999295, + 4.158625136999035, + 4.260530553998251, + 4.367968589998782, + 4.459898264998628, + 4.559621599997627, + 4.663280516000668, + 4.771102959999553, + 4.879523806997895, + 4.991217145998235, + 5.09821530499903, + 5.18762078799773, + 5.291235267999582, + 5.402596641000855, + 5.517916379998496, + 5.624105877999682, + 5.723803016000602, + 5.833960734998982, + 5.954256688000896, + 6.081341667999368, + 6.202840889000072, + 6.302164733999234, + 6.408620941998379, + 6.514813698999205, + 6.614270500998828, + 6.725487329000316, + 6.845529717000318, + 6.951376721000997, + 7.07023564499832, + 7.182928358000936, + 7.292137206997722, + 7.3891275899986795, + 7.494919295000727, + 7.605716654998105, + 7.718513429997984, + 7.824195376000716, + 7.9297832129996095, + 8.048630997000146, + 8.151636127000529, + 8.248458091999055, + 8.3584020720009, + 8.46908400599932, + 8.59001062400057, + 8.723443814000348, + 8.826834906998556, + 8.920594855000672, + 9.017358900000545, + 9.123725919998833, + 9.23983333599972, + 9.335148188998573, + 9.4339634809985, + 9.537985114999174, + 9.625530111999979, + 9.711921883998002, + 9.806165542999224, + 9.912926319997496, + 10.028102315998694, + 10.126468806000048, + 10.233817869000632, + 10.33145148299809, + 10.437272641000163, + 10.54275326199786, + 10.65576732900081, + 10.753243457998906, + 10.853827859999, + 10.979164921998745, + 11.0802276659997, + 11.176660176999576, + 11.264374721999047, + 11.361467044000165, + 11.467297667997627, + 11.571742526997696 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 105.63489150081296, + "p95": 123.35458600136917, + "max": 171.53674599830993 + } + }, + { + "start": 29275.709775189, + "first": 29276.12829346, + "last": 29287.86265546, + "end": 29287.862800609, + "elapsed_s": 12.153025420000631, + "chunk_offsets_s": [ + 0.41851827099890215, + 0.5069875289991614, + 0.61717990300167, + 0.7334281149996968, + 0.8413526020012796, + 0.9418340330012143, + 1.0392967619991396, + 1.1436126150001655, + 1.2506463659992733, + 1.3468960610007343, + 1.4454463780020887, + 1.536422378001589, + 1.631737053001416, + 1.7267342310005915, + 1.8361052760010352, + 1.94872289200066, + 2.0725969539998914, + 2.189217553001072, + 2.294696278000629, + 2.3865354070003377, + 2.4978923900016525, + 2.599625612001546, + 2.689708170000813, + 2.799841836000269, + 2.9050275110021175, + 3.009799388000829, + 3.121615871001268, + 3.2167168449996097, + 3.3245760919999157, + 3.433130268000241, + 3.5346004989987705, + 3.6355745639993984, + 3.7326462480014015, + 3.8281602850001946, + 3.9292033890014864, + 4.0527103999993415, + 4.157508776999748, + 4.259903655998642, + 4.366543852000177, + 4.458741234000627, + 4.558790834002139, + 4.662415383001644, + 4.76989455399962, + 4.878734224999789, + 4.99087377700198, + 5.097465882001416, + 5.187474456000928, + 5.290498932001356, + 5.401765843002067, + 5.51774320000186, + 5.6230514039998525, + 5.723160550001921, + 5.833058786000038, + 5.954065013000218, + 6.08134874100142, + 6.201678482000716, + 6.301023847001488, + 6.407740336999268, + 6.5134797110004, + 6.613270506000845, + 6.725383315999352, + 6.844504779000999, + 6.951429040000221, + 7.06988518900107, + 7.181657473000087, + 7.291366024001036, + 7.388100060001307, + 7.493775511000422, + 7.605346327000007, + 7.717535419000342, + 7.823098871998809, + 7.929350358001102, + 8.048323341001378, + 8.150714082999912, + 8.247663868998643, + 8.357577097998728, + 8.468120698998973, + 8.589645224001288, + 8.723065391001, + 8.825758162001875, + 8.919719769000949, + 9.017073291000997, + 9.122782916001597, + 9.238713152000855, + 9.334128291000525, + 9.433007997999084, + 9.537005088001024, + 9.624651218000508, + 9.710649078999268, + 9.805064126001525, + 9.911734361001436, + 10.027900993001822, + 10.124957206000545, + 10.233170299001358, + 10.330336066999735, + 10.437274881998746, + 10.542528260000836, + 10.654959137998958, + 10.75237622900022, + 10.853471020000143, + 10.97872457899939, + 11.079985734999354, + 11.175963615998626, + 11.263642593999975, + 11.360975678999239, + 11.466309352999815, + 11.570774547999463, + 11.659243734000484, + 11.74832589899961, + 11.842311453001457, + 11.923180281999521, + 12.010964249999233, + 12.087087353000243, + 12.152880271001777 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 104.79837700040662, + "p95": 121.52452500231448, + "max": 133.42016699971282 + } + }, + { + "start": 29275.70946006, + "first": 29275.804038665, + "last": 29286.688585479, + "end": 29286.688679061, + "elapsed_s": 10.979219001001184, + "chunk_offsets_s": [ + 0.09457860499969684, + 0.5075911640014965, + 0.6176045339998382, + 0.7340041499992367, + 0.8421338650005055, + 0.9427283660006651, + 1.0389840480020212, + 1.143995485999767, + 1.2510224860016024, + 1.3472687089997635, + 1.4458145780008635, + 1.536798130000534, + 1.6321345159994962, + 1.7269801610018476, + 1.8363528859990765, + 1.9491278749992489, + 2.0729973609995795, + 2.189684070999647, + 2.2944935300001816, + 2.386797433002357, + 2.498154033000901, + 2.599999188001675, + 2.6898842779992265, + 2.800202980000904, + 2.9053878230006376, + 3.009665246001532, + 3.1216989570020814, + 3.217112852002174, + 3.3251239679993887, + 3.433192123000481, + 3.5346769290008524, + 3.635463462000189, + 3.732462523999857, + 3.8282818500010762, + 3.9292979780002497, + 4.053240291999828, + 4.157994286000758, + 4.259838393001701, + 4.3669969309994485, + 4.4591263299989805, + 4.559161289998883, + 4.662795119002112, + 4.770152291999693, + 4.879170054999122, + 4.990573879000294, + 5.098350583000865, + 5.187715570002183, + 5.290702783000597, + 5.402038668999012, + 5.5174709650018485, + 5.623457843001233, + 5.723714538002241, + 5.833469321001758, + 5.954174626000167, + 6.081870185000298, + 6.202091384999221, + 6.301277616999869, + 6.408006891000696, + 6.513861767001799, + 6.61362846600241, + 6.725058521999017, + 6.84476840500065, + 6.951263220002147, + 7.070239310000034, + 7.1821108069998445, + 7.291724736001925, + 7.388522930999898, + 7.494455176001793, + 7.605627872999321, + 7.717505242999323, + 7.823338689999218, + 7.929892363001272, + 8.048590886999591, + 8.151089659000718, + 8.248065828000108, + 8.358049200000096, + 8.468675151001662, + 8.590012384000147, + 8.722874290000618, + 8.8261445370008, + 8.920159408000472, + 9.017002444001264, + 9.12316556299993, + 9.239093592001154, + 9.334497850999469, + 9.433375013999466, + 9.53728098600186, + 9.625068361001468, + 9.71084669099946, + 9.80522837800163, + 9.912165360001381, + 10.02802201400118, + 10.125002740001946, + 10.234025497000403, + 10.330368720999104, + 10.437533291998989, + 10.542889963999187, + 10.655631683999673, + 10.75280125900099, + 10.853833379998832, + 10.97912541900223 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 105.95956150064012, + "p95": 123.86948600033065, + "max": 413.0125590017997 + } + }, + { + "start": 29275.709123267, + "first": 29275.887483201, + "last": 29288.138767943, + "end": 29288.138836806, + "elapsed_s": 12.429713539000659, + "chunk_offsets_s": [ + 0.17835993400149164, + 0.5084213859990996, + 0.6183543740007735, + 0.7347879890003242, + 0.8418520790000912, + 0.9429205220003496, + 1.0397461440006737, + 1.1448351809995074, + 1.2521580940010608, + 1.3482240170014848, + 1.4468603960012842, + 1.5378639150003437, + 1.6327627589998883, + 1.7275096379999013, + 1.8366015530009463, + 1.9497327739991306, + 2.073447175000183, + 2.1905169490019034, + 2.2952890810011013, + 2.3877096690011967, + 2.499143468001421, + 2.6006910290016094, + 2.690800082000351, + 2.8010794570000144, + 2.9060120659996755, + 3.009828746999119, + 3.122546811002394, + 3.217904435001401, + 3.3259181909998006, + 3.43406151200179, + 3.5346248660025594, + 3.6354842620021373, + 3.733012160002545, + 3.8284381830017082, + 3.930128953001258, + 4.053258531999745, + 4.15862003300208, + 4.260377836999396, + 4.367866832002619, + 4.459862137999153, + 4.559872986999835, + 4.663350339000317, + 4.770956243999535, + 4.879326180001954, + 4.991022430000157, + 5.098493908000819, + 5.187816448000376, + 5.291377729001397, + 5.402756350002164, + 5.518155959001888, + 5.624420447002194, + 5.7239923080014705, + 5.834264937002445, + 5.954404030002479, + 6.081658462000632, + 6.203082389001793, + 6.302481448001345, + 6.4089504389994545, + 6.514736405002623, + 6.614412962000642, + 6.725836794001225, + 6.845374520002224, + 6.951896853999642, + 7.0707294269996055, + 7.1831127219993505, + 7.292542879000393, + 7.389392609002243, + 7.49489331100267, + 7.606624124000518, + 7.717562345002079, + 7.823893655000575, + 7.930053718999261, + 8.049191114001587, + 8.15185311499954, + 8.249082383001223, + 8.358814944000187, + 8.469621291002113, + 8.59055170099964, + 8.723498069000925, + 8.826986664000287, + 8.921099564002361, + 9.017783564002457, + 9.123971323002479, + 9.240433612001652, + 9.335340329002065, + 9.434198132999882, + 9.5379132449998, + 9.625164631001098, + 9.71185327800049, + 9.806673805000173, + 9.912668708999263, + 10.028043997001078, + 10.12633704900145, + 10.23440702500011, + 10.331296526001097, + 10.437078293001832, + 10.543112103001476, + 10.65603086000192, + 10.75373720800053, + 10.854024896001647, + 10.97921663300076, + 11.080905396000162, + 11.176360215002205, + 11.263564787001087, + 11.361959321999166, + 11.466849628999626, + 11.571125083999505, + 11.659724234999885, + 11.7486910430016, + 11.84313916300016, + 11.923691199001041, + 12.011732617000234, + 12.0878200570005, + 12.153290310001466, + 12.203497882001102, + 12.261284898999293, + 12.323508338002284, + 12.373170544000459, + 12.429644676001772 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 104.52379349953844, + "p95": 123.12957899848698, + "max": 330.06145199760795 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 103.46110785995451, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 12.407166782002605, + "end_to_end_tps": 103.16617987732079, + "timing_valid": true, + "steady_tps": 110.23254026009664, + "per_stream_tps": [ + 21.5, + 22.7, + 21.4, + 24.2, + 20.7 + ], + "ttft_s": [ + 0.16, + 0.33, + 0.24, + 0.41, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.1604106109989516, + "decode_tps": 21.549784418740984, + "prefill_tps": null, + "itl_mean_ms": 46.40417651372606, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.37575900022057, + "round_gap_p50_ms": 106.02293799820472, + "round_gap_p95_ms": 119.79857600090327, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3258470879991364, + "decode_tps": 22.71050733283028, + "prefill_tps": null, + "itl_mean_ms": 44.03248176470287, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.09336000037729, + "round_gap_p50_ms": 106.28459100189502, + "round_gap_p95_ms": 120.35579300209065, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.24358076099815662, + "decode_tps": 21.44999449701745, + "prefill_tps": null, + "itl_mean_ms": 46.62005858039016, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.13072500040289, + "round_gap_p50_ms": 105.68749199956073, + "round_gap_p95_ms": 119.86846599756973, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.4085328639994259, + "decode_tps": 24.208375507202334, + "prefill_tps": null, + "itl_mean_ms": 41.308017537256305, + "itl_p50_ms": 0.0, + "itl_p95_ms": 114.87591799959773, + "round_gap_p50_ms": 106.28510400056257, + "round_gap_p95_ms": 119.21760499899392, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08359198100151843, + "decode_tps": 20.692221571990903, + "prefill_tps": null, + "itl_mean_ms": 48.32733868235807, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.49224199936725, + "round_gap_p50_ms": 104.6365850015718, + "round_gap_p95_ms": 119.98825599948759, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 45.33841461568669, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.99579399990034, + "round_gap_p50_ms": 106.02293799820472, + "round_gap_p95_ms": 119.86846599756973, + "round_gap_mean_ms": 105.48627488138783, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 29288.143032705, + "first": 29288.303443316, + "last": 29300.136508327, + "end": 29300.136632033, + "elapsed_s": 11.993599328001437, + "chunk_offsets_s": [ + 0.1604106109989516, + 0.49864418400102295, + 0.6157315630007361, + 0.7332516940005007, + 0.8403503260014986, + 0.9400478440002189, + 1.0323318160008057, + 1.132918195999082, + 1.2419360050007526, + 1.3378105919982772, + 1.4376659459994698, + 1.5295318309981667, + 1.631957917001273, + 1.728116938000312, + 1.838323963998846, + 1.9499101459987287, + 2.064048197000375, + 2.179162772001291, + 2.283841594999103, + 2.3732475629985856, + 2.4842065679986263, + 2.5925564200006193, + 2.686518792001152, + 2.798778199998196, + 2.9023514660002547, + 3.001164831999631, + 3.1086662230009097, + 3.2029812340006174, + 3.3100511000011466, + 3.4175560469993798, + 3.5193949040003645, + 3.628853155998513, + 3.728090174998215, + 3.8213784600011422, + 3.9228199669996684, + 4.036988686999393, + 4.140416913000081, + 4.241624654001498, + 4.347902262001298, + 4.438663581000583, + 4.538933582000027, + 4.647269159999269, + 4.756187687999045, + 4.8645592210014, + 4.970761618998949, + 5.071880108000187, + 5.159308507998503, + 5.260881535999943, + 5.371333226998104, + 5.486530177000532, + 5.599392271000397, + 5.703184019999753, + 5.813583623999875, + 5.9333822000007785, + 6.0529237119990285, + 6.172501005999948, + 6.270923354000843, + 6.378237947999878, + 6.484832048001408, + 6.590980169999966, + 6.70583207099844, + 6.826365591998183, + 6.932246928998211, + 7.041599709998991, + 7.152697587000148, + 7.262991471001442, + 7.358514569001272, + 7.4658898029993, + 7.581847692999872, + 7.698223452000093, + 7.805167431000882, + 7.911190368999087, + 8.02075618500021, + 8.121141160998377, + 8.217215182998189, + 8.3260188259992, + 8.437508254999557, + 8.562799936000374, + 8.69891241300138, + 8.803612185998645, + 8.89754472200002, + 8.987886784998409, + 9.089740780000284, + 9.20636518700121, + 9.301720491999731, + 9.4004348999988, + 9.507108934998541, + 9.603324539999448, + 9.693370380999113, + 9.790059538001515, + 9.897038389000954, + 10.005898053001147, + 10.099129354999604, + 10.206584065999778, + 10.301844715999323, + 10.410095080998872, + 10.514637260999734, + 10.63370432499869, + 10.733001778000471, + 10.833338030999585, + 10.942434936001519, + 11.039658421999775, + 11.134406682998815, + 11.222432562000904, + 11.319435272998817, + 11.424032598999474, + 11.553685650000261, + 11.646425588998682, + 11.736997290001455, + 11.8312212919991, + 11.91208666500097, + 11.993475621999096 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 106.02293799820472, + "p95": 119.79857600090327, + "max": 338.23357300207135 + } + }, + { + "start": 29288.141958599, + "first": 29288.467805687, + "last": 29299.696088537, + "end": 29299.696169257, + "elapsed_s": 11.554210657999647, + "chunk_offsets_s": [ + 0.3258470879991364, + 0.5000098839991551, + 0.6171499339980073, + 0.7332432939983846, + 0.8413340660008544, + 0.9416037640003196, + 1.0333319229976041, + 1.1343273829988902, + 1.2424751779981307, + 1.3387482209982409, + 1.4385383129992988, + 1.5305334740005492, + 1.6331593489994702, + 1.7289916560002894, + 1.8390183650008112, + 1.9514312989995233, + 2.0656594280007994, + 2.1809046729977126, + 2.2854033580006217, + 2.3743911790006678, + 2.485539504999906, + 2.5938478210009634, + 2.687794761000987, + 2.7991870199984987, + 2.9032341229976737, + 3.002427085997624, + 3.1100945660000434, + 3.2039006049999443, + 3.311635023001145, + 3.418998728000588, + 3.5207669640003587, + 3.629712974998256, + 3.7290788520003844, + 3.822116911000194, + 3.9237640629980888, + 4.038482665000629, + 4.141427250000561, + 4.243143736999627, + 4.348834379998152, + 4.439820946998225, + 4.539259147000848, + 4.647756138998375, + 4.757302725000045, + 4.864794659999461, + 4.971582080001099, + 5.072359637000773, + 5.1604439799994, + 5.262702315998467, + 5.372820573000354, + 5.487505765999231, + 5.600579473000835, + 5.704673282998556, + 5.81454396999834, + 5.9348997630004305, + 6.054780448997917, + 6.17444331400111, + 6.272491152998555, + 6.379479782997805, + 6.486298290998093, + 6.592384930998378, + 6.706474283000716, + 6.827207628997712, + 6.933707076997962, + 7.042892467998172, + 7.153500504999101, + 7.263899112000217, + 7.3599088589980965, + 7.4670837730009225, + 7.582842331998108, + 7.699474344000919, + 7.806008938998275, + 7.91229353000017, + 8.02204021599755, + 8.122451725997962, + 8.218358775000524, + 8.327348201997665, + 8.439418104000652, + 8.564141900998948, + 8.700384268999187, + 8.804466809000587, + 8.89796556500005, + 8.989409781999711, + 9.090914280001016, + 9.207649552998191, + 9.302714192999701, + 9.401892985999439, + 9.507976306998899, + 9.604179594000016, + 9.69378524000058, + 9.790190794999944, + 9.897983029000898, + 10.006553921997693, + 10.100033463000727, + 10.207280372000241, + 10.303159416998824, + 10.41077351399872, + 10.516034399999626, + 10.634934514000633, + 10.733529483000893, + 10.833990503000678, + 10.942823081000824, + 11.04031025599761, + 11.135367337999924, + 11.222828893998667, + 11.320419162999315, + 11.425598270998307, + 11.554129937998368 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 106.39201950107235, + "p95": 120.35579300209065, + "max": 174.16279600001872 + } + }, + { + "start": 29288.142576106, + "first": 29288.386156867, + "last": 29300.274271805, + "end": 29300.274342572, + "elapsed_s": 12.131766465998226, + "chunk_offsets_s": [ + 0.24358076099815662, + 0.49854068199783796, + 0.6167272069978935, + 0.7329773799974646, + 0.8404862119969039, + 0.9411787009994441, + 1.0323824549996061, + 1.1334579609974753, + 1.2421689919974597, + 1.3380644429998938, + 1.4378567749990907, + 1.5297400829986145, + 1.6323302459968545, + 1.7281144579974352, + 1.8390205739997327, + 1.9505726929965022, + 2.06478555799913, + 2.1804500899997947, + 2.284666727999138, + 2.373295514997153, + 2.4850499179992767, + 2.5936441749981896, + 2.686782458997186, + 2.79891617499743, + 2.9030034209972655, + 3.001557795996632, + 3.109637997997197, + 3.203333384997677, + 3.3108168649996514, + 3.418287346998113, + 3.52023317199928, + 3.6290295679973497, + 3.7281505719984125, + 3.8216065729975526, + 3.9234659259964246, + 4.037984640999639, + 4.1412432359975355, + 4.242300934998639, + 4.34831746999771, + 4.438949082999898, + 4.5393307029989955, + 4.647075383996707, + 4.7560190509975655, + 4.863914606998151, + 4.970271526999568, + 5.071269534997555, + 5.159343970997725, + 5.261413696996897, + 5.371201948997623, + 5.486848108997947, + 5.600042552996456, + 5.703946418998385, + 5.814201202996628, + 5.9340643559989985, + 6.053932821996568, + 6.173578712998278, + 6.2717407749987615, + 6.378733471996384, + 6.485534480998467, + 6.591971555997588, + 6.706067098999483, + 6.826151143999596, + 6.9328596709965495, + 7.041999719996966, + 7.152830795999762, + 7.262927502997627, + 7.359011882999766, + 7.4659784359973855, + 7.582691435996821, + 7.698395645998971, + 7.805460981999204, + 7.9111484739987645, + 8.021261112997308, + 8.122018008998566, + 8.21800374999657, + 8.32659154799694, + 8.438143617997412, + 8.563390895997145, + 8.699547166997945, + 8.803974034999555, + 8.897092002996942, + 8.988845825999306, + 9.090759721999348, + 9.20689044699975, + 9.302316679997602, + 9.400807902999077, + 9.507300701996428, + 9.603413674998592, + 9.694046845997946, + 9.78985444799764, + 9.897672485996736, + 10.005640382998536, + 10.099357613999018, + 10.206713210998714, + 10.302638087996456, + 10.410102883997752, + 10.515552023996861, + 10.633683036998264, + 10.733128425999894, + 10.83344367799873, + 10.942163761999836, + 11.039975843999855, + 11.134471491997829, + 11.22216566499992, + 11.31984961299895, + 11.424794635997387, + 11.553309002996684, + 11.646942564999335, + 11.737146562998532, + 11.831545342996833, + 11.91263482699651, + 11.993511292999756, + 12.06709327399949, + 12.131695698997646 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 105.68749199956073, + "p95": 119.86846599756973, + "max": 254.95992099968134 + } + }, + { + "start": 29288.142723335, + "first": 29288.551256199, + "last": 29299.084800671, + "end": 29299.0848797, + "elapsed_s": 10.942156364999391, + "chunk_offsets_s": [ + 0.4085328639994259, + 0.4986194000011892, + 0.6162511369984713, + 0.7332465750005213, + 0.8408287169986579, + 0.9404301079994184, + 1.03231405599945, + 1.133507655998983, + 1.2421066099996096, + 1.3375067739980295, + 1.4378304069978185, + 1.529985614000907, + 1.632624128000316, + 1.7280403949989704, + 1.8381558549990586, + 1.950899005998508, + 2.06520282999918, + 2.1796295950007334, + 2.2842342129988538, + 2.37379361299827, + 2.48510317399996, + 2.5938200569980836, + 2.68698082300034, + 2.798474005998287, + 2.9024058149989287, + 3.0016083470000012, + 3.109054140000808, + 3.203079840997816, + 3.310919998999452, + 3.4182929859998694, + 3.5200415100007376, + 3.6291050629988604, + 3.7281126489979215, + 3.8217380409987527, + 3.9225905729981605, + 4.037466490997758, + 4.140927062999253, + 4.241878214997996, + 4.347870298999624, + 4.438733045000845, + 4.538768450998759, + 4.6473792959986895, + 4.756137796997791, + 4.864668590998917, + 4.970039340998483, + 5.072035418001178, + 5.159716857000603, + 5.261366053000529, + 5.371981167998456, + 5.486655593998876, + 5.599850038001023, + 5.703601565997815, + 5.813829612998234, + 5.934303616999387, + 6.0535212219983805, + 6.173179972000071, + 6.271310000000085, + 6.378668511999422, + 6.485779579998052, + 6.591987093997886, + 6.705658079001296, + 6.826225437998801, + 6.932510541999363, + 7.0421061769993685, + 7.1523762100005115, + 7.262875870997959, + 7.3594006649982475, + 7.465868890998536, + 7.582324342998618, + 7.698320136998518, + 7.805068007997761, + 7.911035660999914, + 8.0208127099977, + 8.121619657998963, + 8.217648718000419, + 8.326479198000015, + 8.438414674998057, + 8.563451007001277, + 8.699512176997814, + 8.803608520000125, + 8.897408632001316, + 8.988993426999514, + 9.09023596499901, + 9.206973215998005, + 9.301888678000978, + 9.400913139001204, + 9.50736172300094, + 9.603494242001034, + 9.693813046000287, + 9.789909579998493, + 9.897417653999582, + 10.005613410998194, + 10.099348626998108, + 10.206223544999375, + 10.302596993999032, + 10.40988325500075, + 10.514882225998008, + 10.634068733001186, + 10.733036797999375, + 10.833910212000774, + 10.942077335999784 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 106.36684400014929, + "p95": 119.21760499899392, + "max": 136.0611699965375 + } + }, + { + "start": 29288.14128754, + "first": 29288.224879521, + "last": 29300.548350885, + "end": 29300.548454322, + "elapsed_s": 12.407166782002605, + "chunk_offsets_s": [ + 0.08359198100151843, + 0.5004745630030811, + 0.6175763740029652, + 0.7352115270005015, + 0.8416350840016094, + 0.9416836350028461, + 1.033525136001117, + 1.1345332670025527, + 1.2428852020020713, + 1.3388785870010906, + 1.4388549310024246, + 1.5309418350007036, + 1.6337817530002212, + 1.7301153460030037, + 1.8392310020026343, + 1.9515731370011054, + 2.0658936480031116, + 2.180764500000805, + 2.285534827002266, + 2.3746866030014644, + 2.48603483700208, + 2.59460187300283, + 2.6883209260013246, + 2.79966234099993, + 2.903617945001315, + 3.0029981620027684, + 3.110255011000845, + 3.2048647540032107, + 3.3119354330010538, + 3.4190931290031585, + 3.521350193001126, + 3.6304438580009446, + 3.729130471001554, + 3.822619983002369, + 3.9237108420020377, + 4.0382116440014215, + 4.141842022003402, + 4.244223950001469, + 4.349106446003134, + 4.439987072000804, + 4.540549439003371, + 4.648585612001625, + 4.757059012001264, + 4.86648044300091, + 4.973012591002771, + 5.072392938000121, + 5.1609043030002795, + 5.262970201001735, + 5.373242645000573, + 5.4880151730030775, + 5.601400397001271, + 5.705164361002971, + 5.815432082999905, + 5.935266378000961, + 6.055647058001341, + 6.174365323000529, + 6.272601278000366, + 6.37992073800342, + 6.486524297000869, + 6.592859243002749, + 6.707230712003366, + 6.827218968002853, + 6.933771751002496, + 7.043124740001076, + 7.153486445000453, + 7.264659578002465, + 7.36034303100314, + 7.467175250003493, + 7.58366749200286, + 7.699624603003031, + 7.80657362400234, + 7.91261574100281, + 8.022316347000015, + 8.122989221999887, + 8.218875832000776, + 8.327827110002545, + 8.439057152001624, + 8.564376868002, + 8.701293655001791, + 8.804845224000019, + 8.898236194003402, + 8.989489793002576, + 9.091352325001935, + 9.208006698001554, + 9.303158477003308, + 9.401995511001587, + 9.509056765000423, + 9.604568987000675, + 9.69483615700301, + 9.790659542002686, + 9.899004377002711, + 10.006639188002737, + 10.100473214002704, + 10.207457554002758, + 10.303531507001026, + 10.41215413500322, + 10.516895085002034, + 10.635087169001054, + 10.734027819002222, + 10.835015115000715, + 10.94397860400204, + 11.040827631000866, + 11.135670879000827, + 11.223923885001568, + 11.320958450000035, + 11.425595035001606, + 11.55472321400157, + 11.648029349002172, + 11.738941779000015, + 11.8326850500016, + 11.91364762000012, + 11.995094145000621, + 12.068156373003148, + 12.132898719002696, + 12.183165780003037, + 12.238417962002131, + 12.301462962001096, + 12.348583989001781, + 12.407063345002825 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 104.68876750019263, + "p95": 119.98825599948759, + "max": 416.8825820015627 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.8, + "per_stream_tps": 67.8, + "ttft_s_max": 0.1, + "request": { + "ttft_s": 0.096, + "decode_tps": 67.838, + "prefill_tps": 145.132, + "itl_mean_ms": 14.741, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.062, + "round_gap_p50_ms": 72.707, + "round_gap_p95_ms": 81.181, + "tokens_per_chunk": 4.571 + }, + "pooled": { + "aggregate_prefill_tps": 145.132, + "itl_mean_ms": 14.741, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.062, + "round_gap_p50_ms": 72.707, + "round_gap_p95_ms": 81.181, + "tokens_per_chunk": 4.571 + }, + "timing_valid": true, + "end_to_end_tps": 66.30965934603019, + "makespan_s": 3.860674335002841, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.86430378629716, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8532074639988423, + "end_to_end_tps": 66.43815636501552, + "timing_valid": true, + "per_stream_tps": [ + 67.9 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09563894899838488, + "decode_tps": 67.86430378629716, + "prefill_tps": 146.38387546726835, + "itl_mean_ms": 14.735287098044543, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.43220200063661, + "round_gap_p50_ms": 72.28244900034042, + "round_gap_p95_ms": 81.29898699917248, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.735287098044543, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.43220200063661, + "round_gap_p50_ms": 72.28244900034042, + "round_gap_p95_ms": 81.29898699917248, + "round_gap_mean_ms": 68.31814927275197, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 146.38387546726835 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29314.798352488, + "first": 29314.893991437, + "last": 29318.651489647, + "end": 29318.651559952, + "elapsed_s": 3.8532074639988423, + "chunk_offsets_s": [ + 0.09563894899838488, + 0.14988737200110336, + 0.20243833900167374, + 0.2553114000002097, + 0.307897378999769, + 0.37613523500112933, + 0.4545817860016541, + 0.5272319479990983, + 0.5984078900000895, + 0.6723258849997364, + 0.7446083340000769, + 0.8153055850016244, + 0.8885304680006811, + 0.9727288440008124, + 1.0511801709981228, + 1.1279652679986611, + 1.2035786880005617, + 1.2814398119990074, + 1.3530396839996683, + 1.4089803710012347, + 1.4664772340001946, + 1.5265609189991665, + 1.5865182210000057, + 1.6626392020007188, + 1.7270898979986669, + 1.7762018239991448, + 1.8357289269988541, + 1.9131611289994908, + 1.9944601159986632, + 2.061343119999947, + 2.111835620999045, + 2.1738852150010644, + 2.2529080220010655, + 2.3291986969998106, + 2.406945115999406, + 2.486267741998745, + 2.565446074000647, + 2.6395112389982387, + 2.7150768610008527, + 2.7890589439994073, + 2.8630689499987056, + 2.9477009369984444, + 3.0268645620017196, + 3.103288602000248, + 3.180041882998921, + 3.2495110130003013, + 3.3033574869987206, + 3.3649523079984647, + 3.442863012998714, + 3.5067593429994304, + 3.5669147209991934, + 3.6395563070000208, + 3.701108811001177, + 3.7472675290009647, + 3.7972387349982455, + 3.8531371589997434 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 72.28244900034042, + "p95": 81.29898699917248, + "max": 84.63198699973873 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.83823442960035, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.861041017000389, + "end_to_end_tps": 66.30336193602116, + "timing_valid": true, + "per_stream_tps": [ + 67.8 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.10201770099956775, + "decode_tps": 67.83823442960035, + "prefill_tps": 137.23108698616252, + "itl_mean_ms": 14.740949678425928, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.06156700078282, + "round_gap_p50_ms": 72.70660900030634, + "round_gap_p95_ms": 81.15837999866926, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.740949678425928, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.06156700078282, + "round_gap_p50_ms": 72.70660900030634, + "round_gap_p95_ms": 81.15837999866926, + "round_gap_mean_ms": 68.34440305452021, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 137.23108698616252 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29318.652439973, + "first": 29318.754457674, + "last": 29322.513399842, + "end": 29322.51348099, + "elapsed_s": 3.861041017000389, + "chunk_offsets_s": [ + 0.10201770099956775, + 0.15769606300091255, + 0.2107489859990892, + 0.26038000599874067, + 0.31294932200034964, + 0.379869950000284, + 0.45754189400031464, + 0.5303301829990232, + 0.6013383950012212, + 0.6757588769978611, + 0.7488560829988273, + 0.8199278289976064, + 0.8954396090011869, + 0.9730100769993442, + 1.0486817860000883, + 1.130935352997767, + 1.2099127789988415, + 1.287296487000276, + 1.3590586860009353, + 1.413606723999692, + 1.4723239400009334, + 1.5316490590012108, + 1.5904294770007255, + 1.6649800800005323, + 1.7281380979984533, + 1.7777961370011326, + 1.8360054439981468, + 1.9130670109989296, + 1.9900247670011595, + 2.057473237000522, + 2.111974913001177, + 2.176918258999649, + 2.2544037539992132, + 2.3302718439990713, + 2.407723291998991, + 2.4888816719976603, + 2.5677025039985892, + 2.641379009000957, + 2.7155847049980366, + 2.791968886998802, + 2.8669664459994237, + 2.9439777719999256, + 3.0196288199986157, + 3.10124273999827, + 3.1806502920007915, + 3.2513498879998224, + 3.3044679470003757, + 3.368485323000641, + 3.4468546210009663, + 3.511408203998144, + 3.572728052000457, + 3.6454346610007633, + 3.7077750460011885, + 3.753611694999563, + 3.8037546400009887, + 3.8609598689981794 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 72.70660900030634, + "p95": 81.15837999866926, + "max": 82.25356699767872 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.74487061329869, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.860674335002841, + "end_to_end_tps": 66.30965934603019, + "timing_valid": true, + "per_stream_tps": [ + 67.7 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09646381200218457, + "decode_tps": 67.74487061329869, + "prefill_tps": 145.13214551051485, + "itl_mean_ms": 14.761265184314848, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.4032090010005, + "round_gap_p50_ms": 72.80988399725175, + "round_gap_p95_ms": 81.18124300017371, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.761265184314848, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.4032090010005, + "round_gap_p50_ms": 72.80988399725175, + "round_gap_p95_ms": 81.18124300017371, + "round_gap_mean_ms": 68.43859312727794, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 145.13214551051485 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29322.514306527, + "first": 29322.610770339, + "last": 29326.374892961, + "end": 29326.374980862, + "elapsed_s": 3.860674335002841, + "chunk_offsets_s": [ + 0.09646381200218457, + 0.15120736700191628, + 0.20749761900151498, + 0.2609337659996527, + 0.3168052490000264, + 0.38526410200211103, + 0.46441151900216937, + 0.5356306380017486, + 0.6076529210004082, + 0.6812339990028704, + 0.7540438830001222, + 0.8232313650005381, + 0.8970128660002956, + 0.974844476000726, + 1.0508885400013241, + 1.1272917490023247, + 1.2045980790026078, + 1.287667145999876, + 1.3618248560014763, + 1.4184117069999047, + 1.4751964150018466, + 1.5355956380008138, + 1.5941088630024751, + 1.6701205179997487, + 1.7308110390004003, + 1.7796389280010771, + 1.8383267790013633, + 1.9125276780032436, + 1.9893414690013742, + 2.054572023000219, + 2.104647215001023, + 2.1662029690014606, + 2.2496827989998565, + 2.328973899002449, + 2.409490032001486, + 2.4906712750016595, + 2.5705879690030997, + 2.6446207300032256, + 2.7195885150031245, + 2.7947731780004688, + 2.869778991000203, + 2.9473164310002176, + 3.023415504001605, + 3.098870510002598, + 3.173856526002055, + 3.2496353070018813, + 3.3062483360008628, + 3.3688673060023575, + 3.4466459690011106, + 3.5119675700007065, + 3.572036270001263, + 3.644078248002188, + 3.7072078150013112, + 3.7537878880029893, + 3.803831119003007, + 3.860586434002471 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 72.80988399725175, + "p95": 81.18124300017371, + "max": 83.47982999839587 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.5, + "per_stream_tps": 63.0, + "ttft_s_max": 0.2, + "request": { + "ttft_s": 0.147, + "decode_tps": 63.034, + "prefill_tps": 104.594, + "itl_mean_ms": 15.866, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.683, + "round_gap_p50_ms": 75.691, + "round_gap_p95_ms": 84.626, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 142.273, + "itl_mean_ms": 15.876, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.711, + "round_gap_p50_ms": 75.774, + "round_gap_p95_ms": 84.432, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 122.12295593709948, + "makespan_s": 4.19249596499867, + "steady_tps": 126.5, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.51026922061489, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.19249596499867, + "end_to_end_tps": 122.12295593709948, + "timing_valid": true, + "steady_tps": 126.46067786597926, + "per_stream_tps": [ + 62.3, + 63.7 + ], + "ttft_s": [ + 0.1, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09609396299856598, + "decode_tps": 62.255134610307444, + "prefill_tps": 145.69073397679438, + "itl_mean_ms": 16.062932098044683, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.20872699853498, + "round_gap_p50_ms": 76.78744800068671, + "round_gap_p95_ms": 85.12674499797868, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19152455299990834, + "decode_tps": 63.74126774910824, + "prefill_tps": 73.09767745552028, + "itl_mean_ms": 15.688423454897324, + "itl_p50_ms": 0.0, + "itl_p95_ms": 79.96066800114932, + "round_gap_p50_ms": 75.99140900128987, + "round_gap_p95_ms": 84.43218499814975, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.875677776471004, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.037979001645, + "round_gap_p50_ms": 76.78019299783045, + "round_gap_p95_ms": 84.43218499814975, + "round_gap_mean_ms": 72.29103273214476, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 146.19535491104057 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29326.376173433, + "first": 29326.472267396, + "last": 29330.568315081, + "end": 29330.568411446, + "elapsed_s": 4.192238012998132, + "chunk_offsets_s": [ + 0.09609396299856598, + 0.24764266700003645, + 0.3036225940013537, + 0.36056885899961344, + 0.4163725360012904, + 0.4856090149987722, + 0.5624554599999101, + 0.6385951599986583, + 0.715382607999345, + 0.7935442220004916, + 0.8693593280004279, + 0.9464908330010076, + 1.0271952809998766, + 1.1086104850000993, + 1.1880923339995206, + 1.2680309120005404, + 1.3511617980002484, + 1.438554424999893, + 1.5157120320000104, + 1.5747023809999519, + 1.6266709349984012, + 1.688948764000088, + 1.7565505230013514, + 1.8350274750009703, + 1.9152362019995053, + 1.9815864830015926, + 2.0340036929992493, + 2.0853659630010952, + 2.151731297999504, + 2.2291741330009245, + 2.296321835001436, + 2.3535271959990496, + 2.4159720729985565, + 2.4937741290013946, + 2.57362736000141, + 2.657816097998875, + 2.7420132420011214, + 2.8263846849986294, + 2.9052511689988023, + 2.9855108320007275, + 3.063345056001708, + 3.1418054919995484, + 3.22293449299832, + 3.302038703001017, + 3.3871654479989957, + 3.4710688699997263, + 3.5432446869999694, + 3.595458232000965, + 3.6601243299992348, + 3.7401623090008798, + 3.810582702000829, + 3.867662212000141, + 3.931746715999907, + 4.006334739999147, + 4.067440704999171, + 4.120608711000386, + 4.192141647999961 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.81694650091231, + "p95": 85.12674499797868, + "max": 151.54870400147047 + } + }, + { + "start": 29326.375915481, + "first": 29326.567440034, + "last": 29330.567988015, + "end": 29330.568070013, + "elapsed_s": 4.192154532000131, + "chunk_offsets_s": [ + 0.19152455299990834, + 0.24795443699986208, + 0.30367486300019664, + 0.3610508870006015, + 0.41639124100038316, + 0.4861801680017379, + 0.5629603609995684, + 0.6389517700008582, + 0.715848294999887, + 0.7937407480021648, + 0.8695768990000943, + 0.9468032270015101, + 1.02748829200209, + 1.1090486390021397, + 1.188768568001251, + 1.2678292580021662, + 1.3514904920011759, + 1.4387677129998337, + 1.5160143610009982, + 1.575009296000644, + 1.6267140509989986, + 1.689726355001767, + 1.7564858859986998, + 1.8355412199998682, + 1.915060903000267, + 1.9819055750012922, + 2.0342096940003103, + 2.0856746999997995, + 2.152059226998972, + 2.2293772420016467, + 2.296638769999845, + 2.3538516570006323, + 2.416079169001023, + 2.4939610789988365, + 2.5739217469999858, + 2.6579935240006307, + 2.7424257089987805, + 2.8266785289997642, + 2.9054544200007513, + 2.9858287110000674, + 3.063552349998645, + 3.1418580940007814, + 3.2231296419995488, + 3.3023433500020474, + 3.3873886420005874, + 3.470983631999843, + 3.5435546890003025, + 3.5956763440008217, + 3.6601410020011826, + 3.7403428809993784, + 3.810986383999989, + 3.8682393029994273, + 3.9316427010016923, + 4.00629822799965, + 4.067792833000567, + 4.1203764810015855, + 4.192072533998726 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.38580099956016, + "p95": 84.43218499814975, + "max": 87.27722099865787 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.70230701553903, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.186912605000543, + "end_to_end_tps": 122.28581016678125, + "timing_valid": true, + "steady_tps": 126.8032145189964, + "per_stream_tps": [ + 63.9, + 62.4 + ], + "ttft_s": [ + 0.2, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.1967109569995955, + "decode_tps": 63.91391999866354, + "prefill_tps": 71.17041273928014, + "itl_mean_ms": 15.646043929411784, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.65365400034352, + "round_gap_p50_ms": 75.39508099944214, + "round_gap_p95_ms": 83.95842800018727, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09689784299916937, + "decode_tps": 62.35115350776952, + "prefill_tps": 144.4820603500948, + "itl_mean_ms": 16.038195666667033, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.71155500147142, + "round_gap_p50_ms": 75.34724700235529, + "round_gap_p95_ms": 84.82040200033225, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.842119798039407, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.71063500028686, + "round_gap_p50_ms": 75.39508099944214, + "round_gap_p95_ms": 83.95842800018727, + "round_gap_mean_ms": 72.13822408035801, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 142.27303582625134 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29330.569038009, + "first": 29330.765748966, + "last": 29334.755490168, + "end": 29334.75560545, + "elapsed_s": 4.186567440996441, + "chunk_offsets_s": [ + 0.1967109569995955, + 0.2536772809980903, + 0.3089090749999741, + 0.36120297299930826, + 0.4136617189979006, + 0.482414735997736, + 0.5581317279975337, + 0.6336851339983696, + 0.7108062599982077, + 0.7880795609999041, + 0.8624202559985861, + 0.9389193569986674, + 1.0176200459973188, + 1.0982736999976623, + 1.1815599579967966, + 1.2652962699976342, + 1.3492546979978215, + 1.4315373789977457, + 1.506821343999036, + 1.5673111729993252, + 1.6195248019976134, + 1.68137488299908, + 1.7465822759986622, + 1.825493436997931, + 1.9049981589996605, + 1.9742539139988367, + 2.0261946559985518, + 2.0792264379997505, + 2.144681658999616, + 2.227954193996993, + 2.296521718999429, + 2.349133218998759, + 2.4089185609991546, + 2.4843136419985967, + 2.5651509669987718, + 2.64817472499999, + 2.7330716329997813, + 2.8163723119978386, + 2.8959418089980318, + 2.9766528269974515, + 3.0538597849990765, + 3.133615024999017, + 3.221977354998671, + 3.3034948489985254, + 3.38290602699999, + 3.4621878119978646, + 3.535072943999694, + 3.589093425998726, + 3.650975609998568, + 3.7307418399977905, + 3.800779468998371, + 3.8573018009992666, + 3.921918491996621, + 3.9954715159983607, + 4.058474295998167, + 4.110300415999518, + 4.1864521589996 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 75.47424350013898, + "p95": 83.95842800018727, + "max": 88.36232999965432 + } + }, + { + "start": 29330.568944281, + "first": 29330.665842124, + "last": 29334.755582019, + "end": 29334.755856886, + "elapsed_s": 4.186912605000543, + "chunk_offsets_s": [ + 0.09689784299916937, + 0.2535895889996027, + 0.3087935270013986, + 0.36107770899980096, + 0.41398843899878557, + 0.48227828499875613, + 0.5581658609989972, + 0.6335131080013525, + 0.7107293400003982, + 0.7879169640000327, + 0.8622308750018419, + 0.9387247340018803, + 1.0175882559997262, + 1.0982998110011977, + 1.181447830000252, + 1.2653148520003015, + 1.3492107919992122, + 1.4316765270014002, + 1.5067268560014782, + 1.5671081680011412, + 1.6200156599988986, + 1.6812006780019146, + 1.746951817000081, + 1.8253608480008552, + 1.9048919720007689, + 1.974111080999137, + 2.0261470730001747, + 2.0791959240013966, + 2.1447272500008694, + 2.228118323000672, + 2.2965395960018213, + 2.349399219001498, + 2.4093391390015313, + 2.484245343999646, + 2.5649798400008876, + 2.648018461000902, + 2.732838863001234, + 2.816228481999133, + 2.8958405230005155, + 2.9765511580008024, + 3.0540828080011124, + 3.1336470460009878, + 3.221827942001255, + 3.303899979000562, + 3.382798216000083, + 3.462472513998364, + 3.535373996000999, + 3.58903733400075, + 3.6512691040006757, + 3.730415244001051, + 3.801104080001096, + 3.8571088509997935, + 3.922343505000754, + 3.9955087620000995, + 4.058452583001781, + 4.110460279000108, + 4.186637737999263 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 75.61741150129819, + "p95": 84.82040200033225, + "max": 156.69174600043334 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.48126274974176, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.200897097998677, + "end_to_end_tps": 121.8787292466456, + "timing_valid": true, + "steady_tps": 126.41318120679594, + "per_stream_tps": [ + 62.2, + 63.7 + ], + "ttft_s": [ + 0.1, + 0.2 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.10287265199804096, + "decode_tps": 62.24063137487088, + "prefill_tps": 136.0905909207688, + "itl_mean_ms": 16.066675062742718, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.9736690025602, + "round_gap_p50_ms": 75.60720700348611, + "round_gap_p95_ms": 85.57404200109886, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19802589200116927, + "decode_tps": 63.71732752261823, + "prefill_tps": 70.69782571623178, + "itl_mean_ms": 15.694317996074496, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.73115900333505, + "round_gap_p50_ms": 75.77424100236385, + "round_gap_p95_ms": 84.16987199962023, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.880496529408607, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.73115900333505, + "round_gap_p50_ms": 75.77424100236385, + "round_gap_p95_ms": 84.65386999887414, + "round_gap_mean_ms": 72.31297526784276, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 141.39565143246355 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29334.756882785, + "first": 29334.859755437, + "last": 29338.956757578, + "end": 29338.957336125, + "elapsed_s": 4.2004533399995125, + "chunk_offsets_s": [ + 0.10287265199804096, + 0.25315455899908557, + 0.30756076199759264, + 0.3599278269975912, + 0.41198661499947775, + 0.4813237179987482, + 0.558064889999514, + 0.6333057359988743, + 0.7115223149994563, + 0.7905966459984484, + 0.8646475530003954, + 0.9409435069974279, + 1.0283638859982602, + 1.1118983649976144, + 1.1908027539975592, + 1.2720247369979916, + 1.3529984060005518, + 1.4348582629972952, + 1.5102914689996396, + 1.5703007509982854, + 1.6226541149990226, + 1.6839578750004875, + 1.7509597719981684, + 1.8312314579998201, + 1.911192031999235, + 1.983229321998806, + 2.0380303919992, + 2.0923426109984575, + 2.156363178997708, + 2.233745020999777, + 2.3004513989981206, + 2.352857628997299, + 2.4139365079972777, + 2.4895437150007638, + 2.570167832000152, + 2.652742739999667, + 2.7383167820007657, + 2.82297065199964, + 2.9017412879984477, + 2.9852527500006545, + 3.0693389890002436, + 3.149368582999159, + 3.231173887998011, + 3.311482065000746, + 3.3916187350005202, + 3.4719272479997016, + 3.5438952689983125, + 3.5973825590008346, + 3.660615256998426, + 3.742016521999176, + 3.811515788998804, + 3.867778117000853, + 3.932956322998507, + 4.012262118998478, + 4.0773853229984525, + 4.129678477998823, + 4.199874792997434 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 75.9515805002593, + "p95": 85.57404200109886, + "max": 150.2819070010446 + } + }, + { + "start": 29334.756633507, + "first": 29334.954659399, + "last": 29338.956710488, + "end": 29338.957530605, + "elapsed_s": 4.200897097998677, + "chunk_offsets_s": [ + 0.19802589200116927, + 0.2537335790002544, + 0.3080482170007599, + 0.36036271400007536, + 0.41251958800057764, + 0.4817126220004866, + 0.5580094709985133, + 0.6337837120008771, + 0.7111913519984228, + 0.7909613519987033, + 0.8648298469997826, + 0.941259273000469, + 1.0286788859993976, + 1.1122544270001526, + 1.1904829179984517, + 1.2718458419985836, + 1.3537623749980412, + 1.434359222999774, + 1.5108171610008867, + 1.5706963390002784, + 1.6231367000000319, + 1.6844767420006974, + 1.751345977998426, + 1.8318124360012007, + 1.9115864380000858, + 1.983276349001244, + 2.0383800310009974, + 2.0929063809999207, + 2.1569715269979497, + 2.234286669998255, + 2.301140543000656, + 2.3534426410005835, + 2.4147287939995294, + 2.4902847739977005, + 2.5710159330010356, + 2.6532168429985177, + 2.739165283001057, + 2.8233351550006773, + 2.9019263119989773, + 2.9853874590007763, + 3.069392320998304, + 3.1493898189983156, + 3.231218460998207, + 3.311926793998282, + 3.391642384998704, + 3.4724889780009107, + 3.544296617998043, + 3.5977281240011507, + 3.6606762609990255, + 3.742197261999536, + 3.8117036250005185, + 3.8679659369990986, + 3.9334771979993093, + 4.012805405000108, + 4.077577714000654, + 4.130221528997936, + 4.200076981000166 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.03554500019527, + "p95": 84.16987199962023, + "max": 87.41961299892864 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 212.5, + "per_stream_tps": 54.8, + "ttft_s_max": 0.38, + "request": { + "ttft_s": 0.24, + "decode_tps": 54.749, + "prefill_tps": 60.478, + "itl_mean_ms": 18.266, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.591, + "round_gap_p50_ms": 86.745, + "round_gap_p95_ms": 96.715, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 147.271, + "itl_mean_ms": 18.268, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.617, + "round_gap_p50_ms": 86.848, + "round_gap_p95_ms": 96.396, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 209.06639227751864, + "makespan_s": 4.897965611999098, + "steady_tps": 222.7, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 212.4845139516543, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.897965611999098, + "end_to_end_tps": 209.06639227751864, + "timing_valid": true, + "steady_tps": 222.67934178382015, + "per_stream_tps": [ + 53.1, + 55.3, + 56.5, + 54.2 + ], + "ttft_s": [ + 0.1, + 0.29, + 0.38, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09677377000116394, + "decode_tps": 53.13243063686262, + "prefill_tps": 144.6672998254756, + "itl_mean_ms": 18.820896917639082, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.03706799962674, + "round_gap_p50_ms": 87.37756199843716, + "round_gap_p95_ms": 95.61422399929143, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2853086390023236, + "decode_tps": 55.30645580879128, + "prefill_tps": 49.069667322222166, + "itl_mean_ms": 18.081071827441964, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.052877000242, + "round_gap_p50_ms": 87.50570399934077, + "round_gap_p95_ms": 95.86200799822109, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3790579139968031, + "decode_tps": 56.45694651164678, + "prefill_tps": 36.933670246805804, + "itl_mean_ms": 17.712612207847712, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.37047399781295, + "round_gap_p50_ms": 86.93942700119806, + "round_gap_p95_ms": 95.61949900307809, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19111735299884458, + "decode_tps": 54.1783495930313, + "prefill_tps": 73.25342142052709, + "itl_mean_ms": 18.457557447054924, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.6467429997283, + "round_gap_p50_ms": 87.3706009988382, + "round_gap_p95_ms": 96.06514699771651, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.26803459999592, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.61664400016889, + "round_gap_p50_ms": 87.50570399934077, + "round_gap_p95_ms": 95.61949900307809, + "round_gap_mean_ms": 83.18480041069571, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 147.27078982103447 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29338.962145733, + "first": 29339.058919503, + "last": 29343.858248217, + "end": 29343.859086743, + "elapsed_s": 4.896941009999864, + "chunk_offsets_s": [ + 0.09677377000116394, + 0.4429020200004743, + 0.503873833000398, + 0.5619729210011428, + 0.6195658540018485, + 0.6959965259993623, + 0.7854559039988089, + 0.8797689140010334, + 0.9688807730017288, + 1.0577342750002572, + 1.1436424259991327, + 1.233088969001983, + 1.3251260370016098, + 1.4164284260004933, + 1.5044761179997295, + 1.5952194970013807, + 1.6885904390001087, + 1.7834965469992312, + 1.870781566001824, + 1.9389565710007446, + 1.9958752170023217, + 2.063608306001697, + 2.1346592939989932, + 2.2273685010004556, + 2.315987296999083, + 2.3944361720023153, + 2.455836632001592, + 2.5106412900022406, + 2.58351777200005, + 2.6700789739988977, + 2.7454764649992285, + 2.810174204001669, + 2.876806149000913, + 2.9641837109993503, + 3.0534592920012074, + 3.1450542130005488, + 3.2397275709990936, + 3.335337702999823, + 3.426715466001042, + 3.520625409000786, + 3.608346826000343, + 3.697459536000679, + 3.79307375999997, + 3.886088926999946, + 3.9747381230008614, + 4.0639994779994595, + 4.145143244000792, + 4.202900626001792, + 4.270417482999619, + 4.363535369000601, + 4.44566397699964, + 4.509203006000462, + 4.581763083999249, + 4.665480653002305, + 4.734243999999308, + 4.797765701001481, + 4.89610248399913 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.54948949899699, + "p95": 95.61422399929143, + "max": 346.12824999931036 + } + }, + { + "start": 29338.962349355, + "first": 29339.247657994, + "last": 29343.85833131, + "end": 29343.858491486, + "elapsed_s": 4.8961421310013975, + "chunk_offsets_s": [ + 0.2853086390023236, + 0.442603818999487, + 0.5038328340015141, + 0.5618274650005333, + 0.6191355440023472, + 0.6961771480018797, + 0.7855093870020937, + 0.8791258140008722, + 0.9690040980021877, + 1.057721795001271, + 1.1439004200001364, + 1.2331145480020496, + 1.3251674250022916, + 1.4161706480008434, + 1.5045321159996092, + 1.5957492600027763, + 1.6892161079995276, + 1.783211484002095, + 1.8707171880014357, + 1.9388561470004788, + 1.995803309000621, + 2.0638587640023616, + 2.1345447230014543, + 2.226971111002058, + 2.316017309000017, + 2.394390032000956, + 2.4564950700005284, + 2.5103276280024147, + 2.583218433002912, + 2.6705068789997313, + 2.7452133000006143, + 2.8101507620012853, + 2.876645350999752, + 2.9638245220012323, + 3.05367877900062, + 3.145295423000789, + 3.2394827400021313, + 3.3353447480003524, + 3.426827229999617, + 3.5206650319996697, + 3.608412808000139, + 3.697471024999686, + 3.7929468660004204, + 3.885291162001522, + 3.974938691000716, + 4.063967938000133, + 4.1448899720016925, + 4.202641310999752, + 4.270172091000859, + 4.363852902002691, + 4.445731787000113, + 4.50905036200129, + 4.581947938000667, + 4.665612627999508, + 4.733963793001749, + 4.797515977999865, + 4.895981955000025 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.62673999990511, + "p95": 95.86200799822109, + "max": 157.29517999716336 + } + }, + { + "start": 29338.962668831, + "first": 29339.341726745, + "last": 29343.858442858, + "end": 29343.858818968, + "elapsed_s": 4.896150136999495, + "chunk_offsets_s": [ + 0.3790579139968031, + 0.4421615369974461, + 0.5038127089974296, + 0.5610724679972918, + 0.6187372019994655, + 0.6958045089995721, + 0.7854660379998677, + 0.8791493179996905, + 0.9690792949986644, + 1.0576829109995742, + 1.1433655219989305, + 1.232452795997233, + 1.3244092549975903, + 1.415720610999415, + 1.5045288129986147, + 1.5951921419982682, + 1.6887303619987506, + 1.782635424999171, + 1.8704598449985497, + 1.9388558229984483, + 1.9961467229986738, + 2.0632259379999596, + 2.1345461579985567, + 2.2262176319964055, + 2.3160034299980907, + 2.394296836999274, + 2.4551788619974104, + 2.510561017999862, + 2.583427605997713, + 2.6696691769975587, + 2.7450181699969107, + 2.8095303879999847, + 2.8761765579984058, + 2.963115984999604, + 3.053091616999154, + 3.144462090996967, + 3.239343494999048, + 3.33535654699881, + 3.4264401689979422, + 3.520271113997296, + 3.6079827479989035, + 3.6973489969968796, + 3.7929684959999577, + 3.8853806229999464, + 3.974310648998653, + 4.063352796998515, + 4.14420900299956, + 4.202175999998872, + 4.269760395996855, + 4.363132991999009, + 4.445365920997574, + 4.508285868996609, + 4.58136000399827, + 4.665060739996989, + 4.733780367998406, + 4.7970930759984185, + 4.8957740269979695 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.32553050140268, + "p95": 95.61949900307809, + "max": 98.68095099955099 + } + }, + { + "start": 29338.961474829, + "first": 29339.152592182, + "last": 29343.859269331, + "end": 29343.859440441, + "elapsed_s": 4.897965611999098, + "chunk_offsets_s": [ + 0.19111735299884458, + 0.4437771379998594, + 0.5048356810002588, + 0.5628649499994935, + 0.6201874949983903, + 0.6967649500002153, + 0.7861777010002697, + 0.8806621939984325, + 0.9701175919981324, + 1.05882400399787, + 1.1446446570007538, + 1.2340582389988413, + 1.3257049819985696, + 1.4172954599998775, + 1.5058776559999387, + 1.5962007720008842, + 1.6894967789994553, + 1.7841233500003, + 1.8714939509991382, + 1.9398006219998933, + 1.9967758060010965, + 2.064911855999526, + 2.1355865619989345, + 2.2280001459985215, + 2.3167462929995963, + 2.3956458749999, + 2.4570459999995364, + 2.511408197999117, + 2.584283577998576, + 2.6711211090005236, + 2.746432569998433, + 2.810960996997892, + 2.877561853998486, + 2.964660315999936, + 3.0544721010010107, + 3.14598786300121, + 3.240608599000552, + 3.336446214998432, + 3.4278275899996515, + 3.5214210449994425, + 3.609449167001003, + 3.6982811950001633, + 3.79434634199788, + 3.8862465739985055, + 3.975685461999092, + 4.065009670997824, + 4.145608513001207, + 4.203437875999953, + 4.271160721000342, + 4.364895656999579, + 4.446491153998068, + 4.509741728001245, + 4.582670060000964, + 4.666319850999571, + 4.735027776998322, + 4.798216519000562, + 4.897794501997851 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.6993615001993, + "p95": 96.06514699771651, + "max": 252.65978500101482 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 211.7859617328657, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.912762824998936, + "end_to_end_tps": 208.4366855223103, + "timing_valid": true, + "steady_tps": 221.93244298734254, + "per_stream_tps": [ + 56.3, + 54.0, + 55.1, + 52.9 + ], + "ttft_s": [ + 0.38, + 0.19, + 0.29, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.37977400500312797, + "decode_tps": 56.270006598122706, + "prefill_tps": 36.86402917409971, + "itl_mean_ms": 17.77145695293667, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.02770100071211, + "round_gap_p50_ms": 86.29093600029591, + "round_gap_p95_ms": 96.03244400204858, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19071504200110212, + "decode_tps": 54.01684359955365, + "prefill_tps": 73.40794859756838, + "itl_mean_ms": 18.512744050973446, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.85145900119096, + "round_gap_p50_ms": 86.85655700173811, + "round_gap_p95_ms": 96.95134199864697, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28513702099735383, + "decode_tps": 55.11547056680915, + "prefill_tps": 49.09920132794655, + "itl_mean_ms": 18.143726066673658, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.9817169997259, + "round_gap_p50_ms": 86.78517999942414, + "round_gap_p95_ms": 96.60944099960034, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09646594900186756, + "decode_tps": 52.946490433216425, + "prefill_tps": 145.12893041387034, + "itl_mean_ms": 18.886993109795274, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.82286299983389, + "round_gap_p50_ms": 86.70420999987982, + "round_gap_p95_ms": 96.87837999808835, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.328730045094762, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.77381700283149, + "round_gap_p50_ms": 86.84763799828943, + "round_gap_p95_ms": 96.39576700283214, + "round_gap_mean_ms": 83.46118145534221, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 147.3834522247073 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29343.860859422, + "first": 29344.240633427, + "last": 29348.77235495, + "end": 29348.773030417, + "elapsed_s": 4.912170995001361, + "chunk_offsets_s": [ + 0.37977400500312797, + 0.4406809130014153, + 0.5029892500024289, + 0.5602257620012097, + 0.6170111749997886, + 0.6964274439997098, + 0.7827183800000057, + 0.8737460810007178, + 0.9684009150005295, + 1.056997047002369, + 1.1429197140023462, + 1.2322063250030624, + 1.3229947140025615, + 1.4122980780011858, + 1.499916832002782, + 1.5879701630001364, + 1.68299321500308, + 1.7739586000025156, + 1.857763600000908, + 1.932902788001229, + 1.992264676002378, + 2.0609084560019255, + 2.133862118000252, + 2.225760547000391, + 2.3149570130008215, + 2.393417039002088, + 2.452978034001717, + 2.509994738000387, + 2.5816380310025124, + 2.668485669000802, + 2.742817272002867, + 2.8027146160020493, + 2.8682083260027866, + 2.9601458060024015, + 3.053360744001111, + 3.145473372002016, + 3.2423951710006804, + 3.338427615002729, + 3.4291461020002316, + 3.522828450000816, + 3.6106806400021014, + 3.699953061000997, + 3.790414374001557, + 3.8813603630005673, + 3.9762297160013986, + 4.066666293001617, + 4.146296705999703, + 4.203550452002673, + 4.2717833220012835, + 4.364609853000729, + 4.444472200000746, + 4.50888893299998, + 4.58049180099988, + 4.66447492700172, + 4.735664637002628, + 4.791824833002465, + 4.9114955280019785 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 86.56928699929267, + "p95": 96.03244400204858, + "max": 119.67069499951322 + } + }, + { + "start": 29343.860938822, + "first": 29344.051653864, + "last": 29348.772403597, + "end": 29348.772791656, + "elapsed_s": 4.911852834000456, + "chunk_offsets_s": [ + 0.19071504200110212, + 0.4405594550007663, + 0.50325990300189, + 0.5602198000015051, + 0.6168895890004933, + 0.6964731949992711, + 0.7826775370012911, + 0.8737433800015424, + 0.9686048170005961, + 1.0570012210009736, + 1.142539938999107, + 1.2320255260019621, + 1.3229574850010977, + 1.412394826002128, + 1.5003727520015673, + 1.5884100969997235, + 1.6825913159991615, + 1.7744521820022783, + 1.8577672470019024, + 1.9327808170019125, + 1.9921513369990862, + 2.0607553760019073, + 2.133885540999472, + 2.2256593580023036, + 2.314933691999613, + 2.3931754600016575, + 2.4535055800006376, + 2.510851758001081, + 2.581596054998954, + 2.668452612000692, + 2.742677365000418, + 2.802669666001748, + 2.868003323001176, + 2.960760312002094, + 3.0534642599996005, + 3.1453157190007914, + 3.2422670609994384, + 3.3380001750010706, + 3.4290029639996646, + 3.5227011060014775, + 3.610642469000595, + 3.699826533000305, + 3.790438677999191, + 3.8812280160018418, + 3.9759619290016417, + 4.0665169679996325, + 4.1461201070014795, + 4.203169624001021, + 4.271897263999563, + 4.3639144400003715, + 4.444420681000338, + 4.50870640300127, + 4.580325962000643, + 4.664758103001077, + 4.735520587000792, + 4.791693125000165, + 4.911464774999331 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.3989600004279, + "p95": 96.95134199864697, + "max": 249.8444129996642 + } + }, + { + "start": 29343.860788621, + "first": 29344.145925642, + "last": 29348.772575789, + "end": 29348.772639384, + "elapsed_s": 4.9118507629973465, + "chunk_offsets_s": [ + 0.28513702099735383, + 0.44034908399771666, + 0.5026176959981967, + 0.5598489650001284, + 0.6167261999980838, + 0.696032710999134, + 0.7826154329995916, + 0.8735271429977729, + 0.9685606620005274, + 1.0567852019994461, + 1.1426198189983552, + 1.2316923299986229, + 1.3239801799973065, + 1.4128179069994076, + 1.5003407939984754, + 1.588210366997373, + 1.682943690000684, + 1.7738565299987386, + 1.8582835089982837, + 1.9331918639982177, + 1.992132772997138, + 2.060799364997365, + 2.1338314869972237, + 2.225172479000321, + 2.3156261119984265, + 2.3931603159981023, + 2.4532536339975195, + 2.5102579800004605, + 2.581278404999466, + 2.6680635849988903, + 2.74265824000031, + 2.802361783997185, + 2.8679316039997502, + 2.960460015998251, + 3.053088796998054, + 3.1453992719980306, + 3.242008712997631, + 3.338404480000463, + 3.429346823999367, + 3.5232519899982435, + 3.610376720000204, + 3.6996211169971502, + 3.790340641997318, + 3.8812498309998773, + 3.975972134998301, + 4.066804669997509, + 4.146153893998417, + 4.203393858999334, + 4.272209873997781, + 4.364191590997507, + 4.444440126997506, + 4.508473563997541, + 4.5808712399993965, + 4.664591098000528, + 4.735315500998695, + 4.791568323998945, + 4.911787167999137 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 86.95495500069228, + "p95": 96.60944099960034, + "max": 155.21206300036283 + } + }, + { + "start": 29343.860672182, + "first": 29343.957138131, + "last": 29348.773321374, + "end": 29348.773435007, + "elapsed_s": 4.912762824998936, + "chunk_offsets_s": [ + 0.09646594900186756, + 0.4405443639989244, + 0.5029543849996116, + 0.5602260980012943, + 0.6170682599986321, + 0.6964346060012758, + 0.7828072599986626, + 0.8738683720002882, + 0.9690503230012837, + 1.0570329490001313, + 1.143198952999228, + 1.2321627290002652, + 1.3238548640001682, + 1.4131155940012832, + 1.5005030800020904, + 1.5889190349989804, + 1.682759162998991, + 1.7745258409995586, + 1.8582215290007298, + 1.9329615800015745, + 1.9921549089995096, + 2.0607070730002306, + 2.1336414870020235, + 2.2254643500018574, + 2.315494350001245, + 2.3932361980005226, + 2.4540004539994698, + 2.5109114100014267, + 2.581676312001946, + 2.6683805220018257, + 2.742685157001688, + 2.802535261002049, + 2.868569974998536, + 2.9609421599998313, + 3.053460395000002, + 3.145429960000911, + 3.242308339998999, + 3.33847754800081, + 3.4293999250003253, + 3.5232320329996583, + 3.6106676179988426, + 3.6996709880004346, + 3.790324617999431, + 3.8812531500007026, + 3.975893376999011, + 4.06745079400207, + 4.146535631000006, + 4.204155774001265, + 4.272526705000928, + 4.364560714999243, + 4.444491996000579, + 4.508824734999507, + 4.580846724998992, + 4.664752602999215, + 4.735476942001696, + 4.791725193001184, + 4.912649191999662 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.04584800034354, + "p95": 96.87837999808835, + "max": 344.0784149970568 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 213.2058615341953, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.884319668999524, + "end_to_end_tps": 209.65048755904837, + "timing_valid": true, + "steady_tps": 223.5765954675016, + "per_stream_tps": [ + 55.5, + 56.7, + 54.4, + 53.3 + ], + "ttft_s": [ + 0.29, + 0.38, + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.28861790299924905, + "decode_tps": 55.4978235637084, + "prefill_tps": 48.5070394265751, + "itl_mean_ms": 18.018724623535118, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.3883640023414, + "round_gap_p50_ms": 86.44711299712071, + "round_gap_p95_ms": 96.82574799808208, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3848790800002462, + "decode_tps": 56.68320778399936, + "prefill_tps": 36.375060967177134, + "itl_mean_ms": 17.641909113730186, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.26791300150217, + "round_gap_p50_ms": 85.85258500170312, + "round_gap_p95_ms": 96.94105299786315, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19483331800074666, + "decode_tps": 54.381905147608364, + "prefill_tps": 71.85629307994616, + "itl_mean_ms": 18.38846942352807, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.34145700227236, + "round_gap_p50_ms": 86.46053099801065, + "round_gap_p95_ms": 96.82069400150795, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.10003179400155204, + "decode_tps": 53.30972204073085, + "prefill_tps": 139.95550254534857, + "itl_mean_ms": 18.758304521564725, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.53441799935536, + "round_gap_p50_ms": 86.3772680022521, + "round_gap_p95_ms": 97.22036500170361, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.201851920589522, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.28588400199078, + "round_gap_p50_ms": 86.44039800128667, + "round_gap_p95_ms": 96.82574799808208, + "round_gap_mean_ms": 82.88343285268444, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 145.45318537268153 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29348.775199228, + "first": 29349.063817131, + "last": 29353.65859191, + "end": 29353.658709228, + "elapsed_s": 4.88350999999966, + "chunk_offsets_s": [ + 0.28861790299924905, + 0.44682774300235906, + 0.5079775670019444, + 0.5651159510016441, + 0.6213756640026986, + 0.6985578999992867, + 0.7844661349990929, + 0.873040899001353, + 0.9640676640010497, + 1.057927149002353, + 1.143994655001734, + 1.232512726001005, + 1.3239094160016975, + 1.4138345589999517, + 1.5015161379997153, + 1.590538889002346, + 1.6815548949998629, + 1.7728407790018537, + 1.8581561089995375, + 1.92606231299942, + 1.9876611140025489, + 2.056999510001333, + 2.129080924001755, + 2.220869987999322, + 2.311060529002134, + 2.389975702000811, + 2.451633248001599, + 2.5092801850005344, + 2.581036194002081, + 2.667483306999202, + 2.74213802200029, + 2.80052446800255, + 2.8663657489996694, + 2.9535298600021633, + 3.0503556080002454, + 3.1450890400010394, + 3.241274923002493, + 3.3391435390003608, + 3.43064391200096, + 3.5226838310009043, + 3.610199155002192, + 3.698886300000595, + 3.7894356710021384, + 3.8764668600015284, + 3.9669455350012868, + 4.062761557001068, + 4.144620471000962, + 4.202210502000526, + 4.27117050599918, + 4.362558870001521, + 4.444681660999777, + 4.50871696700051, + 4.579200759999367, + 4.661454954999499, + 4.733208173001913, + 4.789410437999322, + 4.883392682000704 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 86.73915099825535, + "p95": 96.82574799808208, + "max": 158.20984000311 + } + }, + { + "start": 29348.774641262, + "first": 29349.159520342, + "last": 29353.658207166, + "end": 29353.65827999, + "elapsed_s": 4.883638727998914, + "chunk_offsets_s": [ + 0.3848790800002462, + 0.44730836500093574, + 0.5086510379987885, + 0.565765041999839, + 0.621987361999345, + 0.6994513819990971, + 0.7853039670008002, + 0.8736937309986388, + 0.9645395730003656, + 1.058592180001142, + 1.1443573860015022, + 1.2334481789985148, + 1.3252380720005021, + 1.4151389309990918, + 1.5027716049989976, + 1.5908813329988334, + 1.6832150269983686, + 1.7736772559983365, + 1.8587901889986824, + 1.9263942210018286, + 1.9881394479998562, + 2.0579703170005814, + 2.1295449329991243, + 2.2211698710016208, + 2.3118485049999435, + 2.390744876000099, + 2.4524329740015673, + 2.509644385001593, + 2.5815194030001294, + 2.667959801001416, + 2.7429517849996046, + 2.8012313559993345, + 2.867048399999476, + 2.9541409679986828, + 3.051176345001295, + 3.144930957001634, + 3.241872009999497, + 3.3400481630014838, + 3.431839939999918, + 3.523699312001554, + 3.6108541969988437, + 3.6993738249984744, + 3.790508674999728, + 3.876734669000143, + 3.967671097998391, + 4.063103832999332, + 4.145106316998863, + 4.202696396001556, + 4.272021398999641, + 4.363289312001143, + 4.445321405000868, + 4.509088596001675, + 4.5801937360010925, + 4.662204550000752, + 4.733450494000863, + 4.78950170600001, + 4.883565904001443 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 86.0392895010591, + "p95": 96.94105299786315, + "max": 98.17615300198668 + } + }, + { + "start": 29348.774764135, + "first": 29348.969597453, + "last": 29353.658657156, + "end": 29353.658836411, + "elapsed_s": 4.884072275999642, + "chunk_offsets_s": [ + 0.19483331800074666, + 0.4469805240005371, + 0.5087517729989486, + 0.5656759650009917, + 0.6224835799985158, + 0.6999564260004263, + 0.7848120369999378, + 0.8733642119987053, + 0.964550317999965, + 1.0582673579992843, + 1.1441214850019605, + 1.2333816770005797, + 1.3244383860001108, + 1.4143689480006287, + 1.5022169200019562, + 1.5907929130007687, + 1.6823731190015678, + 1.773501051000494, + 1.8590312779997475, + 1.9266832399989653, + 1.9881546399992658, + 2.057681415000843, + 2.12957456300137, + 2.2209544479992473, + 2.311798965998605, + 2.390484400999412, + 2.452375041000778, + 2.509753983998962, + 2.581273764000798, + 2.667738255000586, + 2.742267965000792, + 2.801002951000555, + 2.8667586510018737, + 2.95390399399912, + 3.050724688000628, + 3.145196597000904, + 3.2411816759995418, + 3.339664900999196, + 3.431667793000088, + 3.523682803999691, + 3.610478498001612, + 3.6991607019990624, + 3.790502159001335, + 3.8769626899993455, + 3.9678043290005007, + 4.063391907999176, + 4.145246105999831, + 4.202697283999441, + 4.272162131001096, + 4.3634387259990035, + 4.445278949999192, + 4.509247123001842, + 4.579854070001602, + 4.662130024000362, + 4.733287858998665, + 4.7895304449994, + 4.883893021000404 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 86.46251099889923, + "p95": 96.82069400150795, + "max": 252.14720599979046 + } + }, + { + "start": 29348.774516742, + "first": 29348.874548536, + "last": 29353.657916189, + "end": 29353.658048759, + "elapsed_s": 4.883532017000107, + "chunk_offsets_s": [ + 0.10003179400155204, + 0.4470803580006759, + 0.5085147800018603, + 0.5656886870019662, + 0.6224456720010494, + 0.6997134209996148, + 0.7849891199984995, + 0.873454294000112, + 0.964584109999123, + 1.0584595479995187, + 1.1444239689990354, + 1.2333329409993894, + 1.3249342660019465, + 1.4148704079998424, + 1.502089032001095, + 1.5909299630002351, + 1.6828713649993006, + 1.7734364550015016, + 1.8586798910000653, + 1.9263065759987512, + 1.988026176000858, + 2.0575838180011488, + 2.1296126500019454, + 2.221147068001301, + 2.3115757560008205, + 2.390546932001598, + 2.451978114000667, + 2.5096460269996896, + 2.5812829620008415, + 2.6677660620007373, + 2.7427715070007253, + 2.800332097001956, + 2.8668137329987076, + 2.954040945998713, + 3.0512613110004168, + 3.1449467780003033, + 3.2418577579992416, + 3.3397282939986326, + 3.432099139001366, + 3.5241609449985845, + 3.6105382130008365, + 3.6991916400002083, + 3.7902716690005036, + 3.8772422340007324, + 3.967866058999789, + 4.063340969998535, + 4.145088241999474, + 4.202753782999935, + 4.27203811399886, + 4.3631757740004105, + 4.445272442000714, + 4.5089695589995245, + 4.580019813001854, + 4.662237796001136, + 4.733402267000201, + 4.789537223001389, + 4.883399447000556 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 86.43018400107394, + "p95": 97.22036500170361, + "max": 347.04856399912387 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 243.3, + "per_stream_tps": 50.5, + "ttft_s_max": 0.49, + "request": { + "ttft_s": 0.286, + "decode_tps": 50.516, + "prefill_tps": 48.895, + "itl_mean_ms": 19.796, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.021, + "round_gap_p50_ms": 91.996, + "round_gap_p95_ms": 107.091, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 142.018, + "itl_mean_ms": 19.795, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.001, + "round_gap_p50_ms": 92.101, + "round_gap_p95_ms": 106.385, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 239.84992401370542, + "makespan_s": 5.336670442000468, + "steady_tps": 258.6, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 242.76178889341202, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.353448560003017, + "end_to_end_tps": 239.09821597300987, + "timing_valid": true, + "steady_tps": 258.55911929161175, + "per_stream_tps": [ + 49.6, + 50.5, + 51.5, + 52.5, + 48.6 + ], + "ttft_s": [ + 0.21, + 0.31, + 0.4, + 0.49, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.2125917949997529, + "decode_tps": 49.61118920163412, + "prefill_tps": 65.85390560353598, + "itl_mean_ms": 20.15674318822943, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.38612000001012, + "round_gap_p50_ms": 92.78417799941963, + "round_gap_p95_ms": 107.11569899649476, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3071443539993197, + "decode_tps": 50.54034187784761, + "prefill_tps": 45.581173209620545, + "itl_mean_ms": 19.786174031369406, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.18615200108616, + "round_gap_p50_ms": 93.56126499915263, + "round_gap_p95_ms": 107.17546199884964, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.4015486640018935, + "decode_tps": 51.50208551930934, + "prefill_tps": 34.86501451772725, + "itl_mean_ms": 19.416689439208756, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.56190599728143, + "round_gap_p50_ms": 92.37585700248019, + "round_gap_p95_ms": 107.09088700241409, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.4947975299983227, + "decode_tps": 52.50783087581878, + "prefill_tps": 28.294401550564444, + "itl_mean_ms": 19.044778337254183, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.08267799942405, + "round_gap_p50_ms": 92.12972900058958, + "round_gap_p95_ms": 106.07932700077072, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.10127872100201785, + "decode_tps": 48.56254711983183, + "prefill_tps": 138.23239335458302, + "itl_mean_ms": 20.592000611755864, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.26192200105288, + "round_gap_p50_ms": 93.03316099976655, + "round_gap_p95_ms": 106.3854559979518, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.799277121563527, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.23018399745342, + "round_gap_p50_ms": 93.03316099976655, + "round_gap_p95_ms": 106.3854559979518, + "round_gap_mean_ms": 90.15742260711964, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 141.15455761249143 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29353.660511348, + "first": 29353.873103143, + "last": 29359.013072656, + "end": 29359.013133765, + "elapsed_s": 5.3526224169982015, + "chunk_offsets_s": [ + 0.2125917949997529, + 0.5603807039988169, + 0.6270123450012761, + 0.6878122150010313, + 0.7469777209989843, + 0.8306303259996639, + 0.9215310059989861, + 1.0147573050016945, + 1.1199409699984244, + 1.2176013440002862, + 1.3103855219997058, + 1.4070329720016161, + 1.509436126001674, + 1.6065076279992354, + 1.7009128889985732, + 1.7977194679988315, + 1.90062197899897, + 1.9981701990000147, + 2.094318259998545, + 2.1726773119989957, + 2.2340193259988155, + 2.30888392199995, + 2.3867817700011074, + 2.4829463440000836, + 2.5790153089983505, + 2.6670022520011116, + 2.731890151000698, + 2.792571911999403, + 2.8694248930005415, + 2.9585713720007334, + 3.03962580499865, + 3.1055628719987, + 3.1768758190009976, + 3.267899532998854, + 3.365285652998864, + 3.4625533149992407, + 3.5685404210016713, + 3.6722570349993475, + 3.7729129750005086, + 3.8749628059995302, + 3.9712372469984984, + 4.067633441001817, + 4.174749139998312, + 4.270952586000931, + 4.367228539998905, + 4.463485003998358, + 4.551289530998474, + 4.611746140999458, + 4.685154245999001, + 4.781895773998258, + 4.87189629400018, + 4.941065391998563, + 5.019325605000631, + 5.11089857800107, + 5.187075271998765, + 5.2451589960001, + 5.352561307998258 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.00523850106401, + "p95": 107.11569899649476, + "max": 347.788908999064 + } + }, + { + "start": 29353.660398043, + "first": 29353.967542397, + "last": 29359.013016775, + "end": 29359.013249572, + "elapsed_s": 5.352851528998144, + "chunk_offsets_s": [ + 0.3071443539993197, + 0.5604522799985716, + 0.6269907909991161, + 0.6875160060008056, + 0.7468655009979557, + 0.8306914780005172, + 0.9212317589990562, + 1.0151828699999896, + 1.1201743759993406, + 1.2173605280004267, + 1.3109217929995793, + 1.407966339000268, + 1.5098033219983336, + 1.6064376769973023, + 1.7008689350004715, + 1.7967239979989245, + 1.9009967370002414, + 1.9980512439979066, + 2.093783247000829, + 2.1726486250008747, + 2.233354927997425, + 2.3088294779990974, + 2.386816029000329, + 2.48283901099785, + 2.579983092000475, + 2.667195466998237, + 2.7319461539991607, + 2.792583498998283, + 2.869917312000325, + 2.9585260249987186, + 3.040081819999614, + 3.1058992779981054, + 3.1771708489977755, + 3.268486049000785, + 3.3660764380001638, + 3.462297876998491, + 3.5686938569997437, + 3.672186584000883, + 3.773128861997975, + 3.8743829619997996, + 3.971220949999406, + 4.068124713998259, + 4.175316123000812, + 4.2709693029974005, + 4.367125901997497, + 4.463285026999074, + 4.551693621997401, + 4.611150732998794, + 4.6844395469997835, + 4.782391336000728, + 4.871930902998429, + 4.9412477999976545, + 5.0190647919989715, + 5.111346572997718, + 5.188068841998756, + 5.245443269999669, + 5.352618731998518 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.75618800004304, + "p95": 107.17546199884964, + "max": 253.3079259992519 + } + }, + { + "start": 29353.66082442, + "first": 29354.062373084, + "last": 29359.013628891, + "end": 29359.013736592, + "elapsed_s": 5.352912172002107, + "chunk_offsets_s": [ + 0.4015486640018935, + 0.560117690001789, + 0.6266545379985473, + 0.6876568160005263, + 0.7468856980012788, + 0.8301871760013455, + 0.9202419190005457, + 1.0152330609998899, + 1.119555887999013, + 1.2174190709993127, + 1.3097949280017929, + 1.4070567210001172, + 1.508807368001726, + 1.6063692739990074, + 1.7003679890003696, + 1.7969224199987366, + 1.9000760720009566, + 1.997294041000714, + 2.093645518998528, + 2.1724414320015057, + 2.2333778380016156, + 2.3081241049985692, + 2.3865185659997223, + 2.482828233998589, + 2.5793549430018174, + 2.666967121000198, + 2.7317799699994794, + 2.7920197989988083, + 2.869168419001653, + 2.957963076001761, + 3.0390864350010816, + 3.105528987998696, + 3.176639699999214, + 3.2677799049997702, + 3.3655545780020475, + 3.4619960829986667, + 3.5678055089992995, + 3.6718909929986694, + 3.772198208000191, + 3.874238622000121, + 3.970544289000827, + 4.0675448149995646, + 4.174635702001979, + 4.270447105998755, + 4.3666320529991935, + 4.463012284999422, + 4.5507289550005225, + 4.6117475499995635, + 4.684643378001056, + 4.782091444998514, + 4.871964136000315, + 4.940434050000476, + 5.0188482660014415, + 5.110782070001733, + 5.187372685999435, + 5.244297907000146, + 5.352804471000127 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.18728600192117, + "p95": 107.09088700241409, + "max": 158.56902599989553 + } + }, + { + "start": 29353.661400809, + "first": 29354.156198339, + "last": 29359.012616815, + "end": 29359.013429175, + "elapsed_s": 5.352028366000013, + "chunk_offsets_s": [ + 0.4947975299983227, + 0.5593862979985715, + 0.6265806180017535, + 0.6867370069994649, + 0.7456320769997546, + 0.8294444489984016, + 0.9199689970009786, + 1.0144288279989269, + 1.1191103759992984, + 1.2162006629987445, + 1.3091237139997247, + 1.4062063919991488, + 1.5086359419983637, + 1.6057076520010014, + 1.7001471920011682, + 1.7963043339987053, + 1.899434563001705, + 1.9965887399994244, + 2.0929825070015795, + 2.17190742800085, + 2.2324543750000885, + 2.308078464000573, + 2.385997192999639, + 2.4822958289987582, + 2.5783406180016755, + 2.6660547380015487, + 2.7310392539984605, + 2.7913948530003836, + 2.868785825001396, + 2.9574187110010826, + 3.038468204998935, + 3.104422794000129, + 3.17623839999942, + 3.267247469000722, + 3.3644776529981755, + 3.461379530999693, + 3.567458858000464, + 3.671240528998169, + 3.771730412998295, + 3.873590029001207, + 3.9699125639999693, + 4.066825952999352, + 4.174097653998615, + 4.270155714999419, + 4.366271334998601, + 4.462480935999338, + 4.549789766999311, + 4.611294288999488, + 4.683860495999397, + 4.78081735899832, + 4.871212367001135, + 4.939955319998262, + 5.018175321998569, + 5.110305050999159, + 5.186411014001351, + 5.243983398999262, + 5.35121600599814 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.52639000078489, + "p95": 106.07932700077072, + "max": 107.27170099926298 + } + }, + { + "start": 29353.660288032, + "first": 29353.761566753, + "last": 29359.012526909, + "end": 29359.012652278, + "elapsed_s": 5.352364246002253, + "chunk_offsets_s": [ + 0.10127872100201785, + 0.5603855610024766, + 0.6278911330009578, + 0.6875217440028791, + 0.747056986001553, + 0.8306447750001098, + 0.9206088190003356, + 1.0151186290022451, + 1.1199399750003067, + 1.2171047470001213, + 1.3101379079998878, + 1.407711277002818, + 1.509204691999912, + 1.606367358002899, + 1.7007272770024429, + 1.7968904150002345, + 1.9009669320003013, + 1.9974349000003713, + 2.0939690730010625, + 2.1728288410013192, + 2.2333465780029655, + 2.3084227310027927, + 2.3868182790029095, + 2.482633365001675, + 2.5791268499997386, + 2.667008815002191, + 2.7318828360002954, + 2.7923916670006292, + 2.869467861000885, + 2.9583952680004586, + 3.0394835810002405, + 3.1059330410025723, + 3.1769055469994782, + 3.268250037002872, + 3.3661441110016312, + 3.4623391380009707, + 3.5684785500016005, + 3.672548653001286, + 3.7730164250024245, + 3.87495814200156, + 3.9712011180017726, + 4.068463040002825, + 4.174848496000777, + 4.270870550000836, + 4.367095885001618, + 4.463945112001966, + 4.550792207002814, + 4.611550794001232, + 4.684872001002077, + 4.781744284002343, + 4.871958061001351, + 4.940670043000864, + 5.019252636000601, + 5.110997629999474, + 5.187207372000557, + 5.244993912001519, + 5.352238876999763 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.69653999965522, + "p95": 106.3854559979518, + "max": 459.10684000045876 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 243.4081803102774, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.3355770710004435, + "end_to_end_tps": 239.89907426451896, + "timing_valid": true, + "steady_tps": 258.51433242488764, + "per_stream_tps": [ + 49.6, + 52.5, + 51.5, + 50.5, + 48.7 + ], + "ttft_s": [ + 0.19, + 0.48, + 0.38, + 0.29, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19210890799877234, + "decode_tps": 49.58702048916628, + "prefill_tps": 72.87532965462209, + "itl_mean_ms": 20.16656758432338, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.43995300252573, + "round_gap_p50_ms": 91.52939900013735, + "round_gap_p95_ms": 105.76340500119841, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.47652254100103164, + "decode_tps": 52.493831050051284, + "prefill_tps": 29.379512605196343, + "itl_mean_ms": 19.04985747842504, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.02069999932428, + "round_gap_p50_ms": 92.10097800314543, + "round_gap_p95_ms": 104.83496999950148, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38105927399737993, + "decode_tps": 51.47569471918473, + "prefill_tps": 36.7396910541961, + "itl_mean_ms": 19.426644078439317, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.87267000137945, + "round_gap_p50_ms": 91.8667389996699, + "round_gap_p95_ms": 105.50929199962411, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28632931699758046, + "decode_tps": 50.5156468748404, + "prefill_tps": 48.894748699862625, + "itl_mean_ms": 19.79584667058981, + "itl_p50_ms": 0.0, + "itl_p95_ms": 97.99142200063216, + "round_gap_p50_ms": 91.39671999946586, + "round_gap_p95_ms": 105.84689999814145, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09713592200205312, + "decode_tps": 48.693520813451045, + "prefill_tps": 144.12793651872772, + "itl_mean_ms": 20.536613152930215, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.07979699689895, + "round_gap_p50_ms": 91.99632899981225, + "round_gap_p95_ms": 106.01632299949415, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.795105792941552, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.00127299968153, + "round_gap_p50_ms": 92.10097800314543, + "round_gap_p95_ms": 105.39262800011784, + "round_gap_mean_ms": 90.13842816428743, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 146.8113535616624 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29359.015219251, + "first": 29359.207328159, + "last": 29364.349802893, + "end": 29364.349926956, + "elapsed_s": 5.334707705002074, + "chunk_offsets_s": [ + 0.19210890799877234, + 0.5411881740001263, + 0.6082568960009667, + 0.6722507590020541, + 0.7352377579991298, + 0.8264619990004576, + 0.9159023130014248, + 1.0105918750014098, + 1.1095387109999137, + 1.2048639470012859, + 1.296061226999882, + 1.3937198630010244, + 1.493994151998777, + 1.591550475000986, + 1.6879992399990442, + 1.79008387200156, + 1.8939758359992993, + 1.992415789001825, + 2.0839451880019624, + 2.15898704600113, + 2.218509410999104, + 2.289872422999906, + 2.368457347001822, + 2.4647518940000737, + 2.561548899000627, + 2.648266093001439, + 2.7137693080003373, + 2.7806300010015548, + 2.860491096998885, + 2.9511128939993796, + 3.0326267720010947, + 3.093140549000964, + 3.1604483859991888, + 3.2512938090003445, + 3.3488587499996356, + 3.447631712999282, + 3.5533951180004806, + 3.657483434999449, + 3.763304865002283, + 3.868156200998783, + 3.961915793999651, + 4.058074186999875, + 4.155723995001608, + 4.25060232100077, + 4.347735909999756, + 4.44557863099908, + 4.533649371001957, + 4.594544786999904, + 4.669576903001143, + 4.775082403000852, + 4.868009622001409, + 4.935036697999749, + 5.012582989002112, + 5.1002389210007095, + 5.172427500001504, + 5.230102522000379, + 5.334583642001235 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.22830900034751, + "p95": 105.76340500119841, + "max": 349.079266001354 + } + }, + { + "start": 29359.015418951, + "first": 29359.491941492, + "last": 29364.349655149, + "end": 29364.350716202, + "elapsed_s": 5.335297251000156, + "chunk_offsets_s": [ + 0.47652254100103164, + 0.5412041310009954, + 0.6081498509993253, + 0.6716104869992705, + 0.7354178080022393, + 0.8269094850002148, + 0.9158547490005731, + 1.0111762910019024, + 1.1096537859993987, + 1.2041106760007096, + 1.2963739100014209, + 1.3938930420008546, + 1.4943084660008026, + 1.5917809429993213, + 1.6879756269991049, + 1.7901495920013986, + 1.8939349399988714, + 1.9919753729991498, + 2.0840763510022953, + 2.1588629270008823, + 2.217992662001052, + 2.290488712998922, + 2.3685657259993604, + 2.4648678510020545, + 2.561766915001499, + 2.6482028600003105, + 2.7133589840013883, + 2.7802720069994393, + 2.860525673000666, + 2.9513241669992567, + 3.0325066990008054, + 3.0932094210002106, + 3.161020137002197, + 3.2515098149997357, + 3.3489660599989293, + 3.448137093000696, + 3.5529720630001975, + 3.6574058779988263, + 3.763405492001766, + 3.8676431080020848, + 3.962196593001863, + 4.058099554000364, + 4.155038103999686, + 4.250835630999063, + 4.347922835000645, + 4.445943534999969, + 4.534131684999011, + 4.594670185000723, + 4.669914720001543, + 4.775152542999422, + 4.867992603001767, + 4.935135652001918, + 5.012874808999186, + 5.100179114000639, + 5.171880586000043, + 5.230072582999128, + 5.334236197999417 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.18210600192833, + "p95": 104.83496999950148, + "max": 105.99961400293978 + } + }, + { + "start": 29359.015536373, + "first": 29359.396595647, + "last": 29364.350389887, + "end": 29364.350525782, + "elapsed_s": 5.334989408998808, + "chunk_offsets_s": [ + 0.38105927399737993, + 0.5413020630003302, + 0.6082768979977118, + 0.6721455689985305, + 0.7350417189991276, + 0.8266676210005244, + 0.9155058859978453, + 1.0107145299989497, + 1.109345807999489, + 1.2043330359992979, + 1.2961997749989678, + 1.393846578997909, + 1.4944010119979794, + 1.591717482999229, + 1.6881002779991832, + 1.7898934639997606, + 1.895168360999378, + 1.9922051709982043, + 2.0838943969974935, + 2.1585733169995365, + 2.2176135939989763, + 2.2901385849982034, + 2.368366791997687, + 2.464868940998713, + 2.561538832000224, + 2.6490712970007735, + 2.7136686480007484, + 2.780206997998903, + 2.8610854289981944, + 2.950917930000287, + 3.0326341689979017, + 3.0933468849980272, + 3.161102086000028, + 3.25201795799876, + 3.348488191000797, + 3.4484255440002016, + 3.5529385529989668, + 3.657352827998693, + 3.7632406929988065, + 3.8674494009974296, + 3.96202197599996, + 4.057929910999519, + 4.155266997997387, + 4.250765794000472, + 4.347867099997529, + 4.445739769998909, + 4.533911402999365, + 4.594491059000575, + 4.669443818998843, + 4.774953110998467, + 4.86761837299855, + 4.935280251000222, + 5.012688568000158, + 5.100214263999078, + 5.171879823999916, + 5.230102360997989, + 5.3348535139994055 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.2660004998761, + "p95": 105.50929199962411, + "max": 160.2427890029503 + } + }, + { + "start": 29359.015304631, + "first": 29359.301633948, + "last": 29364.349574849, + "end": 29364.3500936, + "elapsed_s": 5.3347889689976, + "chunk_offsets_s": [ + 0.28632931699758046, + 0.5408580049988814, + 0.6079286130006949, + 0.6715795159980189, + 0.7359514029994898, + 0.8269555000006221, + 0.9163068839989137, + 1.0107135680009378, + 1.1098402329989767, + 1.2048501179997402, + 1.2960128609993262, + 1.3939276249984687, + 1.4939912599984382, + 1.5915763140001218, + 1.6886822909982584, + 1.7900614099999075, + 1.8952716620005958, + 1.992583293998905, + 2.083980013998371, + 2.1586382610003056, + 2.2179646949989547, + 2.29011737899782, + 2.3684184640005697, + 2.464478465000866, + 2.561362452001049, + 2.647994998998911, + 2.7133542000010493, + 2.7801088329979393, + 2.8603335740008333, + 2.9515388199979498, + 3.0324817840009928, + 3.093099411998992, + 3.1604370679997373, + 3.2511535699995875, + 3.3486433930011117, + 3.4478458929988847, + 3.5532385209990025, + 3.6572912629999337, + 3.763138162998075, + 3.867961996998929, + 3.961889735001023, + 4.057761052998103, + 4.155714036998688, + 4.250450265000836, + 4.347581887999695, + 4.445573310000327, + 4.533398564999516, + 4.594397527998808, + 4.669386040000973, + 4.775351895998028, + 4.86829926199789, + 4.935654348999378, + 5.012419083999703, + 5.100085696998576, + 5.172575312000845, + 5.229791113000829, + 5.334270217997982 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.17204299966397, + "p95": 105.84689999814145, + "max": 254.5286880013009 + } + }, + { + "start": 29359.015139131, + "first": 29359.112275053, + "last": 29364.349111407, + "end": 29364.349226101, + "elapsed_s": 5.334086970000499, + "chunk_offsets_s": [ + 0.09713592200205312, + 0.540693287999602, + 0.608040727001935, + 0.6720925970003009, + 0.7359287309991487, + 0.8266586310019193, + 0.9166219610015105, + 1.011714413001755, + 1.1098384399992938, + 1.2042434300019522, + 1.2958971789994393, + 1.39354174500113, + 1.4938176489995385, + 1.5914168220006104, + 1.687827310001012, + 1.7906101219996344, + 1.8952575879993674, + 1.9920911280023574, + 2.0840874570021697, + 2.159370896002656, + 2.218520490001538, + 2.289725114002067, + 2.3688964239991037, + 2.46458293500109, + 2.5612010059994645, + 2.647841003999929, + 2.713687122999545, + 2.780390718999115, + 2.8610792290019162, + 2.9511474130013085, + 3.03325586900246, + 3.093018501000188, + 3.1613458310021088, + 3.251051760002156, + 3.3491315569990547, + 3.447959267999977, + 3.5531930590004777, + 3.657503563001228, + 3.763519886000722, + 3.8685311030021694, + 3.9619170219993975, + 4.057740004001971, + 4.155741277001653, + 4.2503242940001655, + 4.347479053001734, + 4.445417662002001, + 4.533489362002001, + 4.594249714999023, + 4.669129259000329, + 4.775284252002166, + 4.8679462040017825, + 4.934829501999047, + 5.012267623002117, + 5.100029789002292, + 5.172633969999879, + 5.2296365490001335, + 5.333972275999258 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.32914049971441, + "p95": 106.01632299949415, + "max": 443.5573659975489 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 243.34999835743707, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.336670442000468, + "end_to_end_tps": 239.84992401370542, + "timing_valid": true, + "steady_tps": 259.29162163528014, + "per_stream_tps": [ + 52.7, + 51.4, + 50.5, + 49.6, + 48.7 + ], + "ttft_s": [ + 0.49, + 0.38, + 0.28, + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.49254429299980984, + "decode_tps": 52.65084865809374, + "prefill_tps": 28.423839640357798, + "itl_mean_ms": 18.99304618039191, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.6149780001142, + "round_gap_p50_ms": 91.53883100225357, + "round_gap_p95_ms": 107.17824399762321, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.37829977099681855, + "decode_tps": 51.43873803317852, + "prefill_tps": 37.00768827617857, + "itl_mean_ms": 19.440601349025897, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.23656999671948, + "round_gap_p50_ms": 92.07558299749508, + "round_gap_p95_ms": 107.95830499773729, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2837286289977783, + "decode_tps": 50.47975943568885, + "prefill_tps": 49.34292337524256, + "itl_mean_ms": 19.809920078442506, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.10801200021524, + "round_gap_p50_ms": 91.72074900197913, + "round_gap_p95_ms": 107.13748600028339, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19100847500158125, + "decode_tps": 49.56878700927754, + "prefill_tps": 73.29517708512202, + "itl_mean_ms": 20.173985694119065, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.78500100239762, + "round_gap_p50_ms": 91.64375799809932, + "round_gap_p95_ms": 107.53689700140967, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09675320800306508, + "decode_tps": 48.67573555481847, + "prefill_tps": 144.6980445295053, + "itl_mean_ms": 20.54411687058746, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.27451300245593, + "round_gap_p50_ms": 91.12396999989869, + "round_gap_p95_ms": 107.3273629990581, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.792334034513367, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.78500100239762, + "round_gap_p50_ms": 91.72074900197913, + "round_gap_p95_ms": 107.17824399762321, + "round_gap_mean_ms": 90.12580676430194, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 142.01847045963274 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 29364.352134495, + "first": 29364.844678788, + "last": 29369.687905564, + "end": 29369.688038187, + "elapsed_s": 5.335903692001011, + "chunk_offsets_s": [ + 0.49254429299980984, + 0.5584091170021566, + 0.6245374200007063, + 0.6867017250006029, + 0.7483682220008632, + 0.8314414310007123, + 0.9196570170024643, + 1.0140913590003038, + 1.113229800001136, + 1.2079354400011653, + 1.3022397860004276, + 1.4031806210005016, + 1.5106184910000593, + 1.6075480670006073, + 1.7037133270023332, + 1.7983124760030478, + 1.9002558970023529, + 1.9971539400030451, + 2.088771987000655, + 2.1635616300009133, + 2.224421032002283, + 2.2966014810008346, + 2.374772629002109, + 2.4775172830013616, + 2.5756158650001453, + 2.661745825000253, + 2.7257051460001094, + 2.784327741999732, + 2.859710235003149, + 2.9511854100019264, + 3.032360758003051, + 3.092804679003166, + 3.1585815340004046, + 3.250120365002658, + 3.3467519610021554, + 3.4539302049997787, + 3.562821745999827, + 3.6675259210023796, + 3.767210382000485, + 3.8694197309996525, + 3.9643303410011868, + 4.060733102000086, + 4.1593480800002, + 4.254512165000051, + 4.350141867002094, + 4.453495613000996, + 4.544847204000689, + 4.604989981002291, + 4.680583539000509, + 4.777724286999728, + 4.868780394001078, + 4.935032452001906, + 5.011856907000038, + 5.098036118000891, + 5.172897211003146, + 5.231968994001363, + 5.3357710689997475 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 91.57843899993168, + "p95": 107.17824399762321, + "max": 108.89154100004816 + } + }, + { + "start": 29364.351886208, + "first": 29364.730185979, + "last": 29369.687539323, + "end": 29369.688455596, + "elapsed_s": 5.33656938799686, + "chunk_offsets_s": [ + 0.37829977099681855, + 0.558025121998071, + 0.6248540149972541, + 0.6879818929992325, + 0.748726022997289, + 0.8319003029973828, + 0.9205086599977221, + 1.0142105129998527, + 1.1134470829965721, + 1.20914987499782, + 1.3024044789999607, + 1.4030952829998569, + 1.5110535879975941, + 1.6072430089989211, + 1.7036710939974, + 1.7983106329993461, + 1.9005597159994068, + 1.9978399659994466, + 2.0887614139974175, + 2.164010493997921, + 2.224742216996674, + 2.2967593299981672, + 2.3747692509969056, + 2.478167046996532, + 2.575019608997536, + 2.6621730079968984, + 2.726009237998369, + 2.7845210079976823, + 2.860147171999415, + 2.9522227549969102, + 3.0328217439964646, + 3.0929881099982595, + 3.159225189996505, + 3.24995943599788, + 3.3475318839991814, + 3.45399689199985, + 3.5623275629986892, + 3.6676151219980966, + 3.767575860998477, + 3.8695261539978674, + 3.964709091997065, + 4.060928493996471, + 4.159269815998414, + 4.254423268997925, + 4.350611393998406, + 4.453890993998357, + 4.544617670999287, + 4.6056746739996015, + 4.681245173997013, + 4.778057960997103, + 4.868724258998554, + 4.935522318999574, + 5.012833149998187, + 5.098565193999093, + 5.173018135999882, + 5.23195160299656, + 5.335653114998422 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.66509349981789, + "p95": 107.95830499773729, + "max": 179.72535100125242 + } + }, + { + "start": 29364.352209566, + "first": 29364.635938195, + "last": 29369.687467815, + "end": 29369.687784022, + "elapsed_s": 5.335574455999449, + "chunk_offsets_s": [ + 0.2837286289977783, + 0.5578161390003515, + 0.6247851499974786, + 0.6873057699995115, + 0.7482112039979256, + 0.8312017459975323, + 0.9197609830007423, + 1.0136785689974204, + 1.1127865809976356, + 1.208638764997886, + 1.3017233350001334, + 1.4032095789989398, + 1.5103470649992232, + 1.6076495880006405, + 1.7034856649988797, + 1.7983842239991645, + 1.8998361159974593, + 1.9968664119987807, + 2.0880738390005718, + 2.163624295997579, + 2.223605597999267, + 2.2963249860004, + 2.374276209000527, + 2.477957041999616, + 2.5751690590004728, + 2.661117595998803, + 2.7255381639988627, + 2.7839752929976385, + 2.859736188998795, + 2.9514569380007742, + 3.0320201569993515, + 3.092187018999539, + 3.159055862997775, + 3.2499101149987837, + 3.3470679739984917, + 3.4537795649994223, + 3.5619014209987654, + 3.6670135999993363, + 3.767020245999447, + 3.8689429400001245, + 3.9638347619984415, + 4.06029615800071, + 4.158639687997493, + 4.2535759460006375, + 4.349896660998638, + 4.453633851000632, + 4.544108186000813, + 4.605490334997739, + 4.680274754999118, + 4.777535941000679, + 4.8679274459973385, + 4.93505829499918, + 5.01244240899905, + 5.098178210999322, + 5.172451229998842, + 5.231571127998905, + 5.335258249000617 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.40265950211324, + "p95": 107.13748600028339, + "max": 274.08751000257325 + } + }, + { + "start": 29364.352045465, + "first": 29364.54305394, + "last": 29369.687420292, + "end": 29369.688236562, + "elapsed_s": 5.336191097001574, + "chunk_offsets_s": [ + 0.19100847500158125, + 0.5577992040016397, + 0.6245457180011726, + 0.6879170710017206, + 0.7485292860001209, + 0.831463384001836, + 0.9201382110004488, + 1.0141185730026336, + 1.1128492440002447, + 1.2087268510003923, + 1.302400194999791, + 1.4034587140013173, + 1.510995611002727, + 1.6077263520019187, + 1.703765308000584, + 1.7986584150021372, + 1.9002092690025165, + 1.9973896610026713, + 2.088813558999391, + 2.1638724069998716, + 2.224313754002651, + 2.2968743790006556, + 2.3742168990029313, + 2.478175413001736, + 2.5756192049993842, + 2.661446313999477, + 2.7259271170005377, + 2.7846304630002123, + 2.8597418930003187, + 2.9512172600007034, + 3.032393999001215, + 3.0926268520015583, + 3.158927731001313, + 3.2505714889994124, + 3.347018720000051, + 3.4543312980022165, + 3.5625533160018676, + 3.667812989002414, + 3.7679333240012056, + 3.8697972070003743, + 3.9646825849995366, + 4.060602650999499, + 4.159387652001897, + 4.254401655001857, + 4.35080313599974, + 4.4534360930010735, + 4.544698731002427, + 4.605321199000173, + 4.680897120000736, + 4.778200872002344, + 4.868905978000839, + 4.935437017000368, + 5.012026528002025, + 5.0976260930001445, + 5.173025945001427, + 5.232096144001844, + 5.335374827001942 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.65855099874898, + "p95": 107.53689700140967, + "max": 366.7907290000585 + } + }, + { + "start": 29364.351785154, + "first": 29364.448538362, + "last": 29369.687288164, + "end": 29369.687567018, + "elapsed_s": 5.335781864003366, + "chunk_offsets_s": [ + 0.09675320800306508, + 0.5589665170009539, + 0.6247117689999868, + 0.6872032240025874, + 0.7484098730019468, + 0.83156363800299, + 0.9200660470014554, + 1.0142362560000038, + 1.1135107690024597, + 1.2083500510016165, + 1.3022281910016318, + 1.4033431540010497, + 1.5106705170001078, + 1.6075220620004984, + 1.703507164002076, + 1.7988460040032805, + 1.9008119550017, + 1.9977745970027172, + 2.0883583390022977, + 2.1638556140023866, + 2.2244756480031356, + 2.296474628001306, + 2.374811335001141, + 2.4785342540017155, + 2.5749545430007856, + 2.662204569001915, + 2.7258535560031305, + 2.7844617140035552, + 2.860204110002087, + 2.9513280800019857, + 3.032498727003258, + 3.092660041002091, + 3.1594451930031937, + 3.250280960000964, + 3.3478898650027986, + 3.454252778003138, + 3.5631037680032023, + 3.667380240000057, + 3.7680450870029745, + 3.8701275550010905, + 3.9640196100008325, + 4.0605347490018175, + 4.158845163001388, + 4.254065566001373, + 4.350977532001707, + 4.454154908002238, + 4.544254800002818, + 4.605074939001497, + 4.681377729000815, + 4.777923159002967, + 4.868618383003195, + 4.935564210001758, + 5.012550294002722, + 5.097986407003191, + 5.173070388002088, + 5.231936568001402, + 5.335503010002867 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.50105499995698, + "p95": 107.3273629990581, + "max": 462.2133089978888 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.8, + "per_stream_tps": 42.8, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.084, + "decode_tps": 42.83, + "prefill_tps": null, + "itl_mean_ms": 23.348, + "itl_p50_ms": 0.0, + "itl_p95_ms": 72.958, + "round_gap_p50_ms": 59.063, + "round_gap_p95_ms": 78.145, + "tokens_per_chunk": 2.56 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 23.348, + "itl_p50_ms": 0.0, + "itl_p95_ms": 72.958, + "round_gap_p50_ms": 59.063, + "round_gap_p95_ms": 78.145, + "tokens_per_chunk": 2.56 + }, + "timing_valid": true, + "end_to_end_tps": 42.39956986260219, + "makespan_s": 6.037797100998432, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.85798075588761, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.036303507000412, + "end_to_end_tps": 42.410061008879374, + "timing_valid": true, + "per_stream_tps": [ + 42.9 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08633638600076665, + "decode_tps": 42.85798075588761, + "prefill_tps": null, + "itl_mean_ms": 23.332877152935517, + "itl_p50_ms": 0.0, + "itl_p95_ms": 72.95833100215532, + "round_gap_p50_ms": 59.06255900117685, + "round_gap_p95_ms": 78.14529900133493, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.332877152935517, + "itl_p50_ms": 0.0, + "itl_p95_ms": 72.95833100215532, + "round_gap_p50_ms": 59.06255900117685, + "round_gap_p95_ms": 78.14529900133493, + "round_gap_mean_ms": 60.09983509089451, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29386.174011246, + "first": 29386.260347632, + "last": 29392.210231306, + "end": 29392.210314753, + "elapsed_s": 6.036303507000412, + "chunk_offsets_s": [ + 0.08633638600076665, + 0.14623177999965264, + 0.21314405399971292, + 0.28806722900117165, + 0.3443710169995029, + 0.3917263510011253, + 0.43887021000045934, + 0.49178097900221474, + 0.5485397830016154, + 0.6176838659994246, + 0.6845247220007877, + 0.7572607380025147, + 0.8281331370017142, + 0.886142674000439, + 0.9364555009997275, + 0.9924842010004795, + 1.0552599959992222, + 1.1160749920018134, + 1.17470367200076, + 1.23533196200151, + 1.2932459639996523, + 1.3554620899994916, + 1.4281641550005588, + 1.5051733670006797, + 1.5791168190007738, + 1.6436887660020147, + 1.701335433001077, + 1.7526509919989621, + 1.8006859890010674, + 1.845411497000896, + 1.9041516790020978, + 1.9679813690017909, + 2.0132782380023855, + 2.0612552970014804, + 2.1107132280012593, + 2.1602873250012635, + 2.2062309569992067, + 2.264527713999996, + 2.3222668950002117, + 2.3798201999998128, + 2.4377619190017867, + 2.485734677000437, + 2.544263445000979, + 2.6094120720008505, + 2.6591400880024594, + 2.720500068000547, + 2.785862086999259, + 2.845053355998971, + 2.9052113270008704, + 2.9675771100010024, + 3.0282113239991304, + 3.076680653000949, + 3.1286221090012987, + 3.1746570819996123, + 3.218661083999905, + 3.263880368001992, + 3.3250406719998864, + 3.38320792200102, + 3.4410394840015215, + 3.5061927110000397, + 3.555424556001526, + 3.621679703999689, + 3.700312319000659, + 3.770352859999548, + 3.845689567999216, + 3.918647899001371, + 3.986067751000519, + 4.04638410500047, + 4.1279228839994175, + 4.19709043200055, + 4.244860122002137, + 4.288387781001802, + 4.350312172002305, + 4.424657978001051, + 4.487478087001364, + 4.543879319000553, + 4.604418955001165, + 4.6595659619997605, + 4.717319834999216, + 4.7748042560015165, + 4.820166558001802, + 4.8789326709993475, + 4.9560942749994865, + 5.0342395740008214, + 5.107176467001409, + 5.1647920239993255, + 5.223854583000502, + 5.300808322001103, + 5.380390868001996, + 5.459905463001633, + 5.522844570001325, + 5.574380612000823, + 5.638765714000328, + 5.697964570001204, + 5.755341544001567, + 5.812075120000372, + 5.860529125002358, + 5.908058331002394, + 5.967874150999705, + 6.036220059999323 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.06255900117685, + "p95": 78.14529900133493, + "max": 81.53877899894724 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.82983810832773, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.037797100998432, + "end_to_end_tps": 42.39956986260219, + "timing_valid": true, + "per_stream_tps": [ + 42.8 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08394270300050266, + "decode_tps": 42.82983810832773, + "prefill_tps": null, + "itl_mean_ms": 23.34820872940826, + "itl_p50_ms": 0.0, + "itl_p95_ms": 72.79462900260114, + "round_gap_p50_ms": 58.80853299822775, + "round_gap_p95_ms": 77.85134100049618, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.34820872940826, + "itl_p50_ms": 0.0, + "itl_p95_ms": 72.79462900260114, + "round_gap_p50_ms": 58.80853299822775, + "round_gap_p95_ms": 77.85134100049618, + "round_gap_mean_ms": 60.13932551514249, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29392.211274942, + "first": 29392.295217645, + "last": 29398.249010871, + "end": 29398.249072043, + "elapsed_s": 6.037797100998432, + "chunk_offsets_s": [ + 0.08394270300050266, + 0.14293864899809705, + 0.21076244399955613, + 0.286374657000124, + 0.3431164039975556, + 0.39199273199847084, + 0.4392714360001264, + 0.48951588199997786, + 0.5471868889981124, + 0.6159185619981145, + 0.6835284160006267, + 0.7532727260004322, + 0.824102831997152, + 0.8832332780002616, + 0.9332323099988571, + 0.9892638040000747, + 1.0530354639995494, + 1.1144817659987893, + 1.1721953279993613, + 1.234810441997979, + 1.292557107997709, + 1.3543691380000382, + 1.4269214519990783, + 1.5053851029988436, + 1.5828142889986339, + 1.6457143799998448, + 1.701200763000088, + 1.7524501369989594, + 1.8022013179979695, + 1.8466049519993248, + 1.9049055209979997, + 1.9689773699974467, + 2.014121495998552, + 2.0603799839991552, + 2.110348851998424, + 2.1621210159974, + 2.2084852769985446, + 2.270476507997955, + 2.327578196000104, + 2.3848143149989482, + 2.4413470439976663, + 2.48977802599984, + 2.5485865589980676, + 2.6165656960001797, + 2.66690228600055, + 2.724463802998798, + 2.7896096439981193, + 2.848582840997551, + 2.9089371569971263, + 2.9709805229977064, + 3.0310762739973143, + 3.0797962569995434, + 3.1306370069978584, + 3.177051959999517, + 3.2224692439995124, + 3.268708184998104, + 3.3301149910003005, + 3.38792912399731, + 3.446648904999165, + 3.510146923999855, + 3.562447106000036, + 3.628950605998398, + 3.706801946998894, + 3.7766526239975065, + 3.85199232100058, + 3.924530206997588, + 3.990565695999976, + 4.049318358000164, + 4.13099376199898, + 4.201175801998033, + 4.249504561998037, + 4.294657782997092, + 4.3562104319971695, + 4.429005060999771, + 4.4917041919979965, + 4.54934358799801, + 4.611175177000405, + 4.666135997998936, + 4.720578645999922, + 4.778353999998217, + 4.824299375999544, + 4.8845873739992385, + 4.962791514997662, + 5.038939533998928, + 5.112524262000079, + 5.170632195997314, + 5.229583606000233, + 5.307104035997327, + 5.384192940997309, + 5.463463381998736, + 5.527917933999561, + 5.58537174400044, + 5.645622599997296, + 5.702858110998932, + 5.7598666720004985, + 5.816508395997516, + 5.864796738998848, + 5.912655586998881, + 5.971250495000277, + 6.037735928999609 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 58.80853299822775, + "p95": 77.85134100049618, + "max": 81.67540399881545 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.745264756133245, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.049756045998947, + "end_to_end_tps": 42.31575588395958, + "timing_valid": true, + "per_stream_tps": [ + 42.7 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08407372099827626, + "decode_tps": 42.745264756133245, + "prefill_tps": null, + "itl_mean_ms": 23.394404168628206, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.26434700007667, + "round_gap_p50_ms": 59.347150003304705, + "round_gap_p95_ms": 78.16623899998376, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.394404168628206, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.26434700007667, + "round_gap_p50_ms": 59.347150003304705, + "round_gap_p95_ms": 78.16623899998376, + "round_gap_mean_ms": 60.258313767678715, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29398.249814512, + "first": 29398.333888233, + "last": 29404.299461296, + "end": 29404.299570558, + "elapsed_s": 6.049756045998947, + "chunk_offsets_s": [ + 0.08407372099827626, + 0.14298521800083108, + 0.20929631800026982, + 0.285568224000599, + 0.3428240299981553, + 0.3905559609993361, + 0.4378029180006706, + 0.4887122129985073, + 0.5503247759988881, + 0.6189362379991508, + 0.6859133209982247, + 0.7570282449996739, + 0.82709714199882, + 0.8864446279985714, + 0.9376781319988368, + 0.9925353319995338, + 1.0559918719991401, + 1.11781379400054, + 1.1760077180006192, + 1.2390463560004719, + 1.297090333999222, + 1.3589061940001557, + 1.4311475719987357, + 1.5087760839996918, + 1.5891725679975934, + 1.6514410700001463, + 1.7056344769989664, + 1.7565659479987517, + 1.8039864120000857, + 1.8485274039994692, + 1.9060302779980702, + 1.9708902739985206, + 2.0159069979999913, + 2.0621372440000414, + 2.1130362170006265, + 2.1651666239995393, + 2.2105767510001897, + 2.2722794490000524, + 2.330491644999711, + 2.3871287900001334, + 2.444897156998195, + 2.492538639999111, + 2.5558451149991015, + 2.6216479309987335, + 2.6689959689974785, + 2.728343119000783, + 2.79458786000032, + 2.8519835990009597, + 2.9140095689981536, + 2.978436151999631, + 3.0384470610006247, + 3.0868961939995643, + 3.1386469829994894, + 3.1844086599994625, + 3.2307321909975144, + 3.2771553569982643, + 3.336151786999835, + 3.395370915000967, + 3.454205963000277, + 3.5173193869995885, + 3.574005712998769, + 3.637724247000733, + 3.7134159689994704, + 3.7825635349981894, + 3.8580456409981707, + 3.9298203619982814, + 3.9979869029994006, + 4.058335867997812, + 4.140230557000905, + 4.209931486999267, + 4.258282411999971, + 4.302119536998362, + 4.36280873099895, + 4.436073077999026, + 4.498061536000023, + 4.559862122998311, + 4.620989486000326, + 4.6749933530009, + 4.731201087000954, + 4.787394234997919, + 4.831842829000379, + 4.889877384001011, + 4.968043623000995, + 5.044573162998859, + 5.118072155997652, + 5.177812689998973, + 5.238548676999926, + 5.315865364998899, + 5.39463689799959, + 5.473700651000399, + 5.542869476998021, + 5.599449312998331, + 5.658725104000041, + 5.715263103000325, + 5.772358690999681, + 5.8270738949977385, + 5.875880943000084, + 5.922855377000815, + 5.981632512000942, + 6.049646783998469 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.347150003304705, + "p95": 78.16623899998376, + "max": 81.89468900309294 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.3, + "per_stream_tps": 41.0, + "ttft_s_max": 0.17, + "request": { + "ttft_s": 0.126, + "decode_tps": 41.007, + "prefill_tps": null, + "itl_mean_ms": 24.386, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.94, + "round_gap_p50_ms": 58.642, + "round_gap_p95_ms": 79.702, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 24.425, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.101, + "round_gap_p50_ms": 58.533, + "round_gap_p95_ms": 79.886, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 80.54072349931887, + "makespan_s": 6.35703253900283, + "steady_tps": 82.2, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.3356205954062, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.35703253900283, + "end_to_end_tps": 80.54072349931887, + "timing_valid": true, + "steady_tps": 82.20827486650998, + "per_stream_tps": [ + 40.7, + 41.2 + ], + "ttft_s": [ + 0.09, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08637403700049617, + "decode_tps": 40.66826960551655, + "prefill_tps": null, + "itl_mean_ms": 24.58919471371736, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.84632399812108, + "round_gap_p50_ms": 58.75042600018787, + "round_gap_p95_ms": 79.88569000008283, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17050418900180375, + "decode_tps": 41.220122652488925, + "prefill_tps": null, + "itl_mean_ms": 24.25999574117276, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.26600699909613, + "round_gap_p50_ms": 58.83342100059963, + "round_gap_p95_ms": 78.51419500002521, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.42459522744506, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.49155699941912, + "round_gap_p50_ms": 58.75631200251519, + "round_gap_p95_ms": 79.88569000008283, + "round_gap_mean_ms": 61.6660572574108, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29404.301181257, + "first": 29404.387555294, + "last": 29410.657799946, + "end": 29410.657904764, + "elapsed_s": 6.356723507000424, + "chunk_offsets_s": [ + 0.08637403700049617, + 0.23354279699924518, + 0.3025953539981856, + 0.3798390060001111, + 0.4397545249994437, + 0.49517082999955164, + 0.5472503720011446, + 0.604105055997934, + 0.6602912400012428, + 0.73037556700001, + 0.7952318839998043, + 0.8710782079979253, + 0.9444378049993247, + 1.0083416839988786, + 1.0624686449991714, + 1.1216019569983473, + 1.1900171439992846, + 1.2562279419980769, + 1.314971380001225, + 1.3806601719988976, + 1.4427733500015165, + 1.50746719900053, + 1.5857219889985572, + 1.667218407998007, + 1.7439015630006907, + 1.8066786560011678, + 1.86333807399933, + 1.9173295119981049, + 1.9673929560012766, + 2.014488345997961, + 2.0668838230012625, + 2.129352365998784, + 2.184063739001431, + 2.233795570999064, + 2.2856332170013047, + 2.3397829530003946, + 2.39000563800073, + 2.4449981180005125, + 2.5039834240014898, + 2.5625619620004727, + 2.620924979000847, + 2.671841792998748, + 2.7250693579990184, + 2.783026698998583, + 2.8340989309981524, + 2.8875696360009897, + 2.942881860999478, + 2.9970192789987777, + 3.056161758999224, + 3.1226722379979037, + 3.1964639229991008, + 3.2483252130004985, + 3.3007663270000194, + 3.352870693001023, + 3.3998301879983046, + 3.4474105919980502, + 3.507639326999197, + 3.5646270329998515, + 3.618959316998371, + 3.6801253730009194, + 3.731700872998772, + 3.788889385999937, + 3.8626978190004593, + 3.9371893759998784, + 4.017075065999961, + 4.0944201159982185, + 4.167163658999925, + 4.224209467000037, + 4.296568555000704, + 4.37344768299954, + 4.433392836999701, + 4.482098625998333, + 4.543362716998672, + 4.611061526000412, + 4.679066200998932, + 4.733716464001191, + 4.79405395899812, + 4.856227159998525, + 4.911085218998778, + 4.96608075399854, + 5.0262442140010535, + 5.074272197998653, + 5.126770523998857, + 5.197644197000045, + 5.278454324998165, + 5.356799791999947, + 5.421559554000851, + 5.477292846000637, + 5.550883510000858, + 5.633207770999434, + 5.717984864000755, + 5.7867354359987075, + 5.841603316999681, + 5.9030805049987976, + 5.961830930998985, + 6.016677814001014, + 6.074180880998028, + 6.126450392999686, + 6.176682507000805, + 6.229657822997979, + 6.286301456999354, + 6.356618688998424 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.75042600018787, + "p95": 79.88569000008283, + "max": 147.168759998749 + } + }, + { + "start": 29404.30106766, + "first": 29404.471571849, + "last": 29410.657870763, + "end": 29410.658100199, + "elapsed_s": 6.35703253900283, + "chunk_offsets_s": [ + 0.17050418900180375, + 0.2334474000017508, + 0.30242985999939265, + 0.3798866329998418, + 0.4398142900026869, + 0.49521418700169306, + 0.5475678460024938, + 0.6044716600008542, + 0.6606032279996725, + 0.7305324570006633, + 0.7954443970011198, + 0.8711204130013357, + 0.9446056650012906, + 1.0084975339996163, + 1.0626352270010102, + 1.12177044100099, + 1.190194552000321, + 1.2563924599999154, + 1.3150288430006185, + 1.380865328002983, + 1.4429509660003532, + 1.50751800800208, + 1.5860322030021052, + 1.6672024790022988, + 1.7441873870011477, + 1.807052393000049, + 1.8637080360022082, + 1.9173927150004602, + 1.9676296819998242, + 2.0146991960027663, + 2.0670625590028067, + 2.129522993000137, + 2.184223700001894, + 2.2339895320001233, + 2.28579899900069, + 2.3399698930006707, + 2.3901687809993746, + 2.445310746999894, + 2.5041441680004937, + 2.5630199980005273, + 2.6214866020018235, + 2.671890058001736, + 2.7253321680000226, + 2.7831760400003986, + 2.834176896001736, + 2.887623356000404, + 2.9430316020007012, + 2.997081203000562, + 3.0565170730005775, + 3.122731523002585, + 3.196612687999732, + 3.24835591900046, + 3.301102305002132, + 3.352947891002259, + 3.3999798330005433, + 3.4475637870018545, + 3.50757258700105, + 3.5645572380017256, + 3.6191510560020106, + 3.680531433001306, + 3.7315787020015705, + 3.7889131830015685, + 3.8631791900006647, + 3.9370230000022275, + 4.01704651700129, + 4.094868429001508, + 4.167242839001119, + 4.224511252999946, + 4.296712827002921, + 4.373599855000066, + 4.433478254002694, + 4.48214893900149, + 4.54357721300039, + 4.611284722999699, + 4.679131132001203, + 4.733897807000176, + 4.794341000000713, + 4.856485973999952, + 4.911166351001157, + 4.966417900999659, + 5.026321075001761, + 5.074556168001436, + 5.126924887001223, + 5.197707942999841, + 5.2786492310006, + 5.357008835002489, + 5.421898764001526, + 5.477518793002673, + 5.551266045000375, + 5.633268830002635, + 5.718382225000823, + 5.786905328001012, + 5.841802077000466, + 5.903140397000243, + 5.961896709002758, + 6.016718179002055, + 6.074246211002901, + 6.126515771000413, + 6.177046875000087, + 6.2300123070017435, + 6.286648057001003, + 6.356803103000857 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.83342100059963, + "p95": 78.51419500002521, + "max": 85.1133949981886 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.21711449263037, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.363919732000795, + "end_to_end_tps": 80.45356031526012, + "timing_valid": true, + "steady_tps": 82.15009812884833, + "per_stream_tps": [ + 40.6, + 41.2 + ], + "ttft_s": [ + 0.08, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08407013800024288, + "decode_tps": 40.611447184660115, + "prefill_tps": null, + "itl_mean_ms": 24.623599239224433, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.59552699988126, + "round_gap_p50_ms": 58.53262399978121, + "round_gap_p95_ms": 79.51761200092733, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17271170199819608, + "decode_tps": 41.18845067049601, + "prefill_tps": null, + "itl_mean_ms": 24.278650537256482, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.03322499926435, + "round_gap_p50_ms": 58.934928998496616, + "round_gap_p95_ms": 78.88957500108518, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.451124888240457, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.03322499926435, + "round_gap_p50_ms": 58.53262399978121, + "round_gap_p95_ms": 79.51761200092733, + "round_gap_mean_ms": 61.73303808417145, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29410.659743897, + "first": 29410.743814035, + "last": 29417.022831841, + "end": 29417.02296732, + "elapsed_s": 6.363223423002637, + "chunk_offsets_s": [ + 0.08407013800024288, + 0.23811914600082673, + 0.3052377990024979, + 0.3837374400027329, + 0.4428159230010351, + 0.4938250490013161, + 0.5450864970007387, + 0.599423329000274, + 0.6579559530000552, + 0.7260693250027543, + 0.7905572960007703, + 0.8664252820017282, + 0.9390375780021714, + 1.0031062600028235, + 1.0567896160027885, + 1.1178161480020208, + 1.1930760470022506, + 1.2615093500025978, + 1.3212365480030712, + 1.3874343220013543, + 1.449130924000201, + 1.508503651002684, + 1.581926779999776, + 1.6646865560032893, + 1.7405380100026377, + 1.803394702001242, + 1.860706832001597, + 1.9157280340004945, + 1.9649892010020267, + 2.012596699001733, + 2.0662044000018795, + 2.129692218000855, + 2.1878407050026, + 2.239572725000471, + 2.291967179000494, + 2.3475590720008768, + 2.3963729570023133, + 2.4501626820019737, + 2.506765322999854, + 2.562589980003395, + 2.619320534002327, + 2.6695474900006957, + 2.7207151130023703, + 2.78037347800273, + 2.8304732570031774, + 2.8832557840032678, + 2.939208913001494, + 2.9928587780013913, + 3.051149124999938, + 3.1180731439999363, + 3.1971629979998397, + 3.2516631570033496, + 3.3048922140005743, + 3.355267807000928, + 3.4024676570006704, + 3.4494190069999604, + 3.5024323320030817, + 3.5571702779998304, + 3.6099960190003912, + 3.6722088550013723, + 3.7236530450027203, + 3.7814623290032614, + 3.857221625999955, + 3.9302501900019706, + 4.0116519310031435, + 4.089662998001586, + 4.166807628000242, + 4.226149145000818, + 4.298469821002072, + 4.3753893339999195, + 4.437030684002821, + 4.484004668000125, + 4.541222196003218, + 4.608138925003004, + 4.675817825002014, + 4.73039005100145, + 4.790105023999786, + 4.854298756003118, + 4.910149988001649, + 4.966342610001448, + 5.027392053001677, + 5.074227904002328, + 5.129023479003081, + 5.205619006002962, + 5.287096729000041, + 5.366614341000968, + 5.431186919002357, + 5.486448514002404, + 5.554104405000544, + 5.63323509300244, + 5.7171000560010725, + 5.786131857003056, + 5.8420504050009185, + 5.902102330001071, + 5.9621789490011, + 6.015867821002757, + 6.073460872001306, + 6.128887935003149, + 6.182290443000966, + 6.238078835001943, + 6.292952751002304, + 6.363087944002473 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.53262399978121, + "p95": 79.51761200092733, + "max": 154.04900800058385 + } + }, + { + "start": 29410.659511103, + "first": 29410.832222805, + "last": 29417.023278692, + "end": 29417.023430835, + "elapsed_s": 6.363919732000795, + "chunk_offsets_s": [ + 0.17271170199819608, + 0.23811765799837303, + 0.30561398599820677, + 0.3840684479982883, + 0.44311959799961187, + 0.49383465999926557, + 0.5448752279990003, + 0.5993506080012594, + 0.658285536999756, + 0.7260797829985677, + 0.7908856489993923, + 0.8669188739986566, + 0.9394207869991078, + 1.003419851000217, + 1.0568223529990064, + 1.1178616309989593, + 1.1934302919980837, + 1.2614513100015756, + 1.3211986589994922, + 1.387857146000897, + 1.4491215829984867, + 1.5085371720015246, + 1.5820241119981802, + 1.6648232959996676, + 1.7407146680016012, + 1.8032083170000988, + 1.8610268529992027, + 1.9157879269987461, + 1.9650129010005912, + 2.0128976390005846, + 2.066199296998093, + 2.12968444299986, + 2.187954383000033, + 2.239957133999269, + 2.292332796001574, + 2.347605242001009, + 2.3964238129992737, + 2.4505424999988463, + 2.5075009249994764, + 2.563096128000325, + 2.6198083060007775, + 2.6700354700005846, + 2.721218750000844, + 2.7802985009984695, + 2.8308233759998984, + 2.88328303399976, + 2.9392605050015845, + 2.993408027999976, + 3.0518922109986306, + 3.118462542999623, + 3.1972180119992117, + 3.2516670250006428, + 3.3048730679984146, + 3.3552044880016183, + 3.402184975999262, + 3.449702066998725, + 3.502948444998765, + 3.5573561640012485, + 3.6102756729997054, + 3.6723888070009707, + 3.7239319949985656, + 3.7816269120012294, + 3.8573130410004524, + 3.9304250729983323, + 4.01162256599855, + 4.089445664998493, + 4.166788529000769, + 4.2262872270002845, + 4.298429292000947, + 4.375832326000818, + 4.437389679998887, + 4.48415148300046, + 4.541609579999204, + 4.608577455001068, + 4.676162938998459, + 4.7314525859983405, + 4.79068477400142, + 4.854726762998325, + 4.910743971999182, + 4.96688496799834, + 5.028099140999984, + 5.074606682999729, + 5.129337805999967, + 5.205531760999293, + 5.287164248999034, + 5.366731792000792, + 5.431672852999327, + 5.486743106001086, + 5.554681964000338, + 5.633571539001423, + 5.717439893000119, + 5.78650935800033, + 5.842407738000475, + 5.902582891001657, + 5.962565116999031, + 6.016403638001066, + 6.073789656998997, + 6.1292009509998024, + 6.182656188000692, + 6.238471977001609, + 6.29331574599928, + 6.3637675889985985 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.934928998496616, + "p95": 78.88957500108518, + "max": 83.86835399869597 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.6517616253806, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.329359965999174, + "end_to_end_tps": 80.89285532035211, + "timing_valid": true, + "steady_tps": 82.52841018998076, + "per_stream_tps": [ + 41.4, + 40.8 + ], + "ttft_s": [ + 0.17, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.1660685579990968, + "decode_tps": 41.38111568960153, + "prefill_tps": null, + "itl_mean_ms": 24.16561234116956, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.1009929990687, + "round_gap_p50_ms": 58.4333200022229, + "round_gap_p95_ms": 80.04723500198452, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08320875700155739, + "decode_tps": 40.8258808126903, + "prefill_tps": null, + "itl_mean_ms": 24.494266384306897, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.9870539988915, + "round_gap_p50_ms": 58.48295000032522, + "round_gap_p95_ms": 80.28422300048987, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.32993936273823, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.1009929990687, + "round_gap_p50_ms": 58.447670002351515, + "round_gap_p95_ms": 80.08971399976872, + "round_gap_mean_ms": 61.42707462869553, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29417.024876815, + "first": 29417.190945373, + "last": 29423.35317652, + "end": 29423.353245083, + "elapsed_s": 6.3283682679975755, + "chunk_offsets_s": [ + 0.1660685579990968, + 0.22770984599992516, + 0.2946115619997727, + 0.37229747499804944, + 0.43073079500027234, + 0.481764801999816, + 0.5324722109980939, + 0.5871465629970771, + 0.6451913439996133, + 0.7139244110003347, + 0.7824923829975887, + 0.8625396179995732, + 0.9375453639986517, + 1.001762752999639, + 1.05566164999982, + 1.1131066009984352, + 1.1792204220000713, + 1.2455252809995727, + 1.3044960209990677, + 1.3699713429996336, + 1.4323425509974186, + 1.4914823389990488, + 1.565992470998026, + 1.6479509469972982, + 1.7249098699976457, + 1.791840735997539, + 1.8513719239999773, + 1.905071772998781, + 1.9553371939982753, + 2.001757591999194, + 2.0542657659971155, + 2.1174230619981245, + 2.1716901599975245, + 2.220949620997999, + 2.27374437799881, + 2.3278823149994423, + 2.3791951829989557, + 2.4343059589991753, + 2.4904395839985227, + 2.545354234000115, + 2.602657111998269, + 2.6529865699994843, + 2.7051425719982944, + 2.7641582699980063, + 2.8197412349982187, + 2.875813595997897, + 2.9311905199974717, + 2.9844453730001987, + 3.043209676998231, + 3.1077606769977137, + 3.1784469300000637, + 3.23040162399775, + 3.283245913000428, + 3.3328919899977336, + 3.3803011319978395, + 3.4283731809991878, + 3.4817511990004277, + 3.5352230649987177, + 3.588259592997929, + 3.649034047997702, + 3.7009822959989833, + 3.759323513997515, + 3.8402256349982054, + 3.915326627997274, + 3.997264511999674, + 4.0742545729990525, + 4.146348917998694, + 4.201242940998782, + 4.272920067996893, + 4.350185118997615, + 4.4100845719985955, + 4.456622276000417, + 4.513396313999692, + 4.5807496790002915, + 4.6468891669974255, + 4.701793393000116, + 4.761795747999713, + 4.831474735998199, + 4.888504844999261, + 4.943629762998171, + 5.005272544000036, + 5.052880780996929, + 5.105580084000394, + 5.174197258998902, + 5.2542869729986705, + 5.332872047998535, + 5.397794823998993, + 5.45429451399832, + 5.5217155309983355, + 5.599856885997724, + 5.6832963899978495, + 5.752112938000209, + 5.813570236998203, + 5.877052484996966, + 5.936190869997517, + 5.990764076999767, + 6.04796424199958, + 6.100504275000276, + 6.149243805997685, + 6.203673648000404, + 6.2579623159981566, + 6.328299704997335 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.4333200022229, + "p95": 80.04723500198452, + "max": 83.43950400012545 + } + }, + { + "start": 29417.024465117, + "first": 29417.107673874, + "last": 29423.353711802, + "end": 29423.353825083, + "elapsed_s": 6.329359965999174, + "chunk_offsets_s": [ + 0.08320875700155739, + 0.2279917910018412, + 0.2949691060020996, + 0.37250797800152213, + 0.4310029680018488, + 0.4819261580014427, + 0.5331914109992795, + 0.5873133249988314, + 0.6452979390014661, + 0.7141790390014648, + 0.7829789930001425, + 0.8632632160006324, + 0.9377357160010433, + 1.0024765000016487, + 1.0559221000003163, + 1.1137224529993546, + 1.17994182899929, + 1.2455263049996574, + 1.3046444700012216, + 1.3710212350015354, + 1.4325930689992674, + 1.4919445919986174, + 1.566092221000872, + 1.64817234200018, + 1.725175757001125, + 1.792515731001913, + 1.8519975480012363, + 1.9055590390016732, + 1.9554964380004094, + 2.001889568000479, + 2.054500227000972, + 2.117787227998633, + 2.1718624390014156, + 2.221275419000449, + 2.273771357999067, + 2.3279873579995183, + 2.3793752509991464, + 2.434365565000917, + 2.490781103999325, + 2.5457204630001797, + 2.6028306859989243, + 2.653085566998925, + 2.705242673000612, + 2.7644589580013417, + 2.8202941700001247, + 2.8763312810006028, + 2.931364220999967, + 2.9849579719993926, + 3.043405642001744, + 3.1082686230001855, + 3.178291440999601, + 3.2305680319987005, + 3.283268221999606, + 3.333406092999212, + 3.3807895819991245, + 3.4285650189995067, + 3.4820036029996118, + 3.5354501360016, + 3.588403339999786, + 3.649054804998741, + 3.701118093998957, + 3.7596010439992824, + 3.8407719640017604, + 3.9158339349996822, + 3.9972883720001846, + 4.074275425999076, + 4.1466233450009895, + 4.2012762049998855, + 4.2732226440020895, + 4.350281012000778, + 4.410365140000067, + 4.456866764001461, + 4.513601586000732, + 4.580970000999514, + 4.647107345001132, + 4.70202161600173, + 4.76248883199878, + 4.832110453000496, + 4.889109423002083, + 4.94437126000048, + 5.005430618999526, + 5.053042439001729, + 5.105688980002014, + 5.174879021000379, + 5.254362330000731, + 5.332994521999353, + 5.397753575001843, + 5.454423145001783, + 5.521702301000914, + 5.599761835001118, + 5.683435465001821, + 5.752811108999595, + 5.8137089279989596, + 5.877346886998566, + 5.936269922000065, + 5.9909938179989695, + 6.0482190450020425, + 6.1006957450008485, + 6.14947489000042, + 6.203794249999191, + 6.258058945000812, + 6.329246684999816 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.48295000032522, + "p95": 80.28422300048987, + "max": 144.7830340002838 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 141.8, + "per_stream_tps": 36.0, + "ttft_s_max": 0.35, + "request": { + "ttft_s": 0.208, + "decode_tps": 36.006, + "prefill_tps": null, + "itl_mean_ms": 27.774, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.277, + "round_gap_p50_ms": 65.546, + "round_gap_p95_ms": 92.036, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 27.718, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.202, + "round_gap_p50_ms": 65.544, + "round_gap_p95_ms": 92.381, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 140.68992255146784, + "makespan_s": 7.278417540001101, + "steady_tps": 146.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 141.82516566549168, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.278417540001101, + "end_to_end_tps": 140.68992255146784, + "timing_valid": true, + "steady_tps": 146.07783316658055, + "per_stream_tps": [ + 36.3, + 36.7, + 35.5, + 35.9 + ], + "ttft_s": [ + 0.25, + 0.34, + 0.09, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.25372552299813833, + "decode_tps": 36.30777450324091, + "prefill_tps": null, + "itl_mean_ms": 27.54231052940846, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.16056999744615, + "round_gap_p50_ms": 65.22077200133936, + "round_gap_p95_ms": 91.8576430012763, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33796086000074865, + "decode_tps": 36.7447811148553, + "prefill_tps": null, + "itl_mean_ms": 27.214749133332482, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.3458690000698, + "round_gap_p50_ms": 65.17573499877471, + "round_gap_p95_ms": 91.3314729987178, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08556934299849672, + "decode_tps": 35.45858091560702, + "prefill_tps": null, + "itl_mean_ms": 28.201918243148075, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.28047599995625, + "round_gap_p50_ms": 65.08350900185178, + "round_gap_p95_ms": 91.63608499875409, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1695981259981636, + "decode_tps": 35.87397683298751, + "prefill_tps": null, + "itl_mean_ms": 27.875359474516397, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.37282199924812, + "round_gap_p50_ms": 64.8315390026255, + "round_gap_p95_ms": 91.50628299903474, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.708584345101354, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.23839500069153, + "round_gap_p50_ms": 65.09534899669234, + "round_gap_p95_ms": 91.63608499875409, + "round_gap_mean_ms": 69.95731691089945, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29423.3558473, + "first": 29423.609572823, + "last": 29430.632862008, + "end": 29430.633327259, + "elapsed_s": 7.27747995900063, + "chunk_offsets_s": [ + 0.25372552299813833, + 0.4045331060005992, + 0.49069367599804536, + 0.5825513189993217, + 0.6497326030003023, + 0.7035950399986177, + 0.7587581560001126, + 0.8210240799999156, + 0.8822505449970777, + 0.9609086839991505, + 1.0322941659978824, + 1.1181728720002866, + 1.199809859997913, + 1.274183257999539, + 1.3364705329986464, + 1.4001489609981945, + 1.4805623600004765, + 1.5600143699994078, + 1.625378090000595, + 1.7022686759992212, + 1.7709568020000006, + 1.8360976379990461, + 1.9218528259989398, + 2.01232335199893, + 2.0959597660003055, + 2.1679708899973775, + 2.230644470997504, + 2.291110480997304, + 2.3461486479973246, + 2.3980767849971016, + 2.4637639339998714, + 2.534719678998954, + 2.594015317998128, + 2.6482154019977315, + 2.705701880997367, + 2.767930042999069, + 2.8260249919985654, + 2.888238616000308, + 2.9512203249978484, + 3.0142042089973984, + 3.0816895549978653, + 3.1385546409983363, + 3.1967427029994724, + 3.2626959269982763, + 3.3171988789981697, + 3.373372873000335, + 3.4347437119977258, + 3.498185372998705, + 3.5650808840000536, + 3.6392886149988044, + 3.722758823998447, + 3.779350106997299, + 3.83745001999705, + 3.895794025000214, + 3.9471986329990614, + 4.000880403997144, + 4.057762799999182, + 4.116529032999097, + 4.175061552999978, + 4.2442005919983785, + 4.30031221599711, + 4.365532987998449, + 4.455490729997109, + 4.546530537998478, + 4.638805072998366, + 4.726767097999982, + 4.810871306999616, + 4.8741497339979105, + 4.95523984600004, + 5.039750271997036, + 5.107065993997821, + 5.159386928000458, + 5.224482276997151, + 5.2976208359978045, + 5.372735356999328, + 5.436815765999199, + 5.506120562997239, + 5.578493821998563, + 5.638980977997562, + 5.7030180000001565, + 5.771045508998213, + 5.823827162999805, + 5.882077645997924, + 5.96192072699705, + 6.053869899998972, + 6.144471559000522, + 6.221001780999359, + 6.284396360999381, + 6.35865507500057, + 6.448855430000549, + 6.552835178998066, + 6.630278475000523, + 6.6935061369986215, + 6.7632388999991235, + 6.82806404099756, + 6.886824973997136, + 6.952053343000443, + 7.011621290999756, + 7.065371394997783, + 7.123341079997772, + 7.1836943899979815, + 7.277014707997296 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.22077200133936, + "p95": 91.8576430012763, + "max": 150.8075830024609 + } + }, + { + "start": 29423.355326668, + "first": 29423.693287528, + "last": 29430.633048557, + "end": 29430.633515983, + "elapsed_s": 7.278189315002237, + "chunk_offsets_s": [ + 0.33796086000074865, + 0.40473287900022115, + 0.49107874800029094, + 0.5833988880003744, + 0.6501688380012638, + 0.7042294810016756, + 0.7593278560016188, + 0.8214680720011529, + 0.8828169020016503, + 0.9614845889991557, + 1.0327022050005326, + 1.1189083749995916, + 1.2004688670022006, + 1.2745210060020327, + 1.3368303360002756, + 1.400470538999798, + 1.480534305999754, + 1.5600665230012964, + 1.6257656410016352, + 1.702518907000922, + 1.7721459259992116, + 1.8361617219998152, + 1.9227151100021729, + 2.0126934700019774, + 2.0965857459996187, + 2.1682503260017256, + 2.2310356840025634, + 2.2918087520010886, + 2.3467510379996384, + 2.3985489570004574, + 2.4642276289996516, + 2.5351745139996638, + 2.594386475000647, + 2.6490263790001336, + 2.706323174999852, + 2.7684062289990834, + 2.826494557000842, + 2.8887081810025848, + 2.9516944640017755, + 3.014641338999354, + 3.082049996999558, + 3.1388136679997842, + 3.1972124599997187, + 3.2632568949993583, + 3.3176798800013785, + 3.3738448530020833, + 3.4350872809991415, + 3.4987483080003585, + 3.5656509830005234, + 3.6398612890006916, + 3.7228424829991127, + 3.779960686002596, + 3.838071635000233, + 3.8960504140013654, + 3.9476550040017173, + 4.001337942001555, + 4.058120219000557, + 4.11730976599938, + 4.175505800001702, + 4.244778672000393, + 4.300943506001204, + 4.3661192409999785, + 4.455954665001627, + 4.546996952001791, + 4.639697616999911, + 4.727442818999407, + 4.811337785002252, + 4.874593612999888, + 4.955857542001468, + 5.040181021002354, + 5.107541221001156, + 5.1598684240016155, + 5.2250523770017026, + 5.298185051000473, + 5.3734224010004255, + 5.437542106999899, + 5.506762714001525, + 5.5787701530025515, + 5.639546934999089, + 5.703577128002507, + 5.771485646000656, + 5.824384260002262, + 5.882550889000413, + 5.962538294999831, + 6.054058476001956, + 6.145389949000673, + 6.221481165001023, + 6.284962829999131, + 6.359061753999413, + 6.4495183559993166, + 6.5532870550014195, + 6.631158897002024, + 6.694306766999944, + 6.763809656000376, + 6.828317518000404, + 6.887517568000476, + 6.9524794690005365, + 7.011978133999946, + 7.066013736999594, + 7.123787294000067, + 7.184154470000067, + 7.2777218890005315 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.17573499877471, + "p95": 91.3314729987178, + "max": 103.76869900210295 + } + }, + { + "start": 29423.35552569, + "first": 29423.441095033, + "last": 29430.632584185, + "end": 29430.632658235, + "elapsed_s": 7.277132545001223, + "chunk_offsets_s": [ + 0.08556934299849672, + 0.4054406809991633, + 0.4907854289995157, + 0.5827281890015001, + 0.649716192001506, + 0.7045288129993423, + 0.7589111160013999, + 0.8208789070013154, + 0.8827524970001832, + 0.961579893999442, + 1.0320848599985766, + 1.1182994910013804, + 1.2001991059987631, + 1.2740578849989106, + 1.336424758999783, + 1.3996501740002714, + 1.4799972789987805, + 1.559700834000978, + 1.6250404519996664, + 1.702090123999369, + 1.7710940719989594, + 1.83574630899966, + 1.9220267849996162, + 2.012588505000167, + 2.0959055150015047, + 2.167636147001758, + 2.2307259240005806, + 2.2912562759993307, + 2.3463171369985503, + 2.398540559999674, + 2.4636240690015256, + 2.5353128249989823, + 2.5942796070012264, + 2.648410824000166, + 2.70564419099901, + 2.7680986919986026, + 2.8261305300002277, + 2.8886131639992527, + 2.951422575999459, + 3.0146196839996264, + 3.081586780001089, + 3.138371706001635, + 3.1966172169995843, + 3.2628020900010597, + 3.3173464969986526, + 3.373534948001179, + 3.4351671840013296, + 3.498289120001573, + 3.565267860998574, + 3.6393611429994053, + 3.7224615830018593, + 3.7795318540011067, + 3.83811566499935, + 3.8955198479998216, + 3.9473462349997135, + 4.000641012000415, + 4.057832688999042, + 4.116647670001839, + 4.175134783999965, + 4.2447586490015965, + 4.30111454100188, + 4.365728458000376, + 4.4556136050014175, + 4.546667950999108, + 4.639458868001384, + 4.727881424998486, + 4.811014589999104, + 4.874313568001526, + 4.955189734999294, + 5.039923830998305, + 5.107465699999011, + 5.159606323999469, + 5.225093305001792, + 5.298221004999505, + 5.373284953999246, + 5.436894195001514, + 5.5063172959999065, + 5.578682398001547, + 5.639038407000044, + 5.703160674998799, + 5.771802510000271, + 5.824042273001396, + 5.882149230001232, + 5.962153909000335, + 6.053497761000472, + 6.145133845999226, + 6.221142040998529, + 6.284604385000421, + 6.358936174001428, + 6.449025869998877, + 6.552736480000021, + 6.631150629000331, + 6.6940562300005695, + 6.763678046998393, + 6.828575717998319, + 6.8869895429998, + 6.952559853998537, + 7.011864517000504, + 7.065873234998435, + 7.124018113998318, + 7.184213935001026, + 7.277058495001256 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.08350900185178, + "p95": 91.63608499875409, + "max": 319.8713380006666 + } + }, + { + "start": 29423.355098443, + "first": 29423.524696569, + "last": 29430.632913235, + "end": 29430.633068532, + "elapsed_s": 7.2779700889987, + "chunk_offsets_s": [ + 0.1695981259981636, + 0.40555186699930346, + 0.491790261999995, + 0.5836972920005792, + 0.6500697099982062, + 0.7047950390006008, + 0.7597538070003793, + 0.8213895269982459, + 0.8834339419991011, + 0.9615729520010063, + 1.03269985299994, + 1.1190726749991882, + 1.2010159359997488, + 1.2746606440014148, + 1.3374754920005216, + 1.4003148429983412, + 1.4813694010008476, + 1.560890422999364, + 1.6258073829994828, + 1.7032051040005172, + 1.7720844789982948, + 1.8369160180009203, + 1.9231255160011642, + 2.013146721998055, + 2.0964514490005968, + 2.1683290760011005, + 2.231463234998955, + 2.2920879800003604, + 2.3470829170000798, + 2.3990077749986085, + 2.463642956001422, + 2.5356289660012408, + 2.594878880001488, + 2.6492039209988434, + 2.706354312998883, + 2.768820138000592, + 2.826878829000634, + 2.889123049000773, + 2.9520715239996207, + 3.0147925880010007, + 3.0825052010004583, + 3.138718250000238, + 3.1972543939991738, + 3.2633321419998538, + 3.318007535999641, + 3.373922851998941, + 3.4356559900006687, + 3.4988411640006234, + 3.5657384820005973, + 3.640158797999902, + 3.7228098710002087, + 3.780383471999812, + 3.8387119469989557, + 3.8960102050004934, + 3.9480563100005384, + 4.001379347999318, + 4.05845927900009, + 4.117373641998711, + 4.176026176999585, + 4.245283280000876, + 4.300938306998432, + 4.3663955419979175, + 4.456294012001308, + 4.547360145999846, + 4.639803187998041, + 4.728151423998497, + 4.81167195099988, + 4.875119221000205, + 4.955742002999614, + 5.040754737001407, + 5.108091697999043, + 5.160371529000258, + 5.224950592000823, + 5.2990524230008305, + 5.373237035000784, + 5.437215710000601, + 5.507043557001452, + 5.579169173000992, + 5.6395683490009105, + 5.704109788999631, + 5.7720443600010185, + 5.82475580299797, + 5.883048380997934, + 5.962893539999641, + 6.0543998229986755, + 6.145325812998635, + 6.221243148000212, + 6.285391468998569, + 6.359039233000658, + 6.449675746000139, + 6.553355136999016, + 6.6311316739993345, + 6.694407811999554, + 6.764407778999157, + 6.82874463799817, + 6.887812630000553, + 6.95307607200084, + 7.012540366999019, + 7.06620532099987, + 7.124214622999716, + 7.184581063000223, + 7.277814791999845 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 64.8315390026255, + "p95": 91.50628299903474, + "max": 235.95374100113986 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 140.88209033351893, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.325259999997797, + "end_to_end_tps": 139.7902600044651, + "timing_valid": true, + "steady_tps": 145.35356534543482, + "per_stream_tps": [ + 36.6, + 35.6, + 36.1, + 35.2 + ], + "ttft_s": [ + 0.35, + 0.17, + 0.27, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.35009122799965553, + "decode_tps": 36.564092057040966, + "prefill_tps": null, + "itl_mean_ms": 27.349236470578106, + "itl_p50_ms": 0.0, + "itl_p95_ms": 85.77450399752706, + "round_gap_p50_ms": 65.5472100006591, + "round_gap_p95_ms": 91.8308469990734, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16841492200182984, + "decode_tps": 35.63421518858754, + "prefill_tps": null, + "itl_mean_ms": 28.0629163490113, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.27342399995541, + "round_gap_p50_ms": 65.54437900194898, + "round_gap_p95_ms": 92.49934299805318, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2678818599997612, + "decode_tps": 36.137254569734, + "prefill_tps": null, + "itl_mean_ms": 27.672273721577323, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.03300899994792, + "round_gap_p50_ms": 65.22710100034601, + "round_gap_p95_ms": 92.21365099801915, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08457086499765865, + "decode_tps": 35.220870837225405, + "prefill_tps": null, + "itl_mean_ms": 28.39225652941797, + "itl_p50_ms": 0.0, + "itl_p95_ms": 85.81581500038737, + "round_gap_p50_ms": 65.67679100044188, + "round_gap_p95_ms": 92.46398500181385, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.869170767646175, + "itl_p50_ms": 0.0, + "itl_p95_ms": 85.77942600095412, + "round_gap_p50_ms": 65.54437900194898, + "round_gap_p95_ms": 92.38169499803917, + "round_gap_mean_ms": 70.36275787871064, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29430.635111782, + "first": 29430.98520301, + "last": 29437.95925831, + "end": 29437.9595548, + "elapsed_s": 7.324443017998419, + "chunk_offsets_s": [ + 0.35009122799965553, + 0.4168257769997581, + 0.49284338199868216, + 0.5829546530003427, + 0.6513837629972841, + 0.7054323159973137, + 0.7601045509982214, + 0.8217530649999389, + 0.8838553619971208, + 0.9649902860001021, + 1.035633682997286, + 1.1210252979981306, + 1.2101091739968979, + 1.2874779269986902, + 1.3495666149974568, + 1.4151138249981159, + 1.4910212169997976, + 1.567668709998543, + 1.6327142189984443, + 1.7102316159980546, + 1.779912116999185, + 1.8444033689993375, + 1.9312209130002884, + 2.0236026079983276, + 2.1082889969984535, + 2.18613415299842, + 2.2550270409992663, + 2.314933886998915, + 2.369700515999284, + 2.4209430619994237, + 2.4808825990003243, + 2.549924753999221, + 2.6101487839987385, + 2.664885250000225, + 2.721738813997945, + 2.783801735000452, + 2.840069390997087, + 2.9028133659994637, + 2.96672664399739, + 3.029840389997844, + 3.0965183779990184, + 3.1520229339985235, + 3.2171865569980582, + 3.2853650149991154, + 3.339563718000136, + 3.3979396859976987, + 3.459201245997974, + 3.5176810749981087, + 3.582946128997719, + 3.657819207997818, + 3.742340486998728, + 3.7988668459984183, + 3.857958349999535, + 3.916405568998016, + 3.9691360629985866, + 4.02166978799869, + 4.080150400997809, + 4.13858480899944, + 4.202968842997507, + 4.275722457998199, + 4.331820114999573, + 4.397719631997461, + 4.483750544997747, + 4.569001802999992, + 4.66139290099818, + 4.750092840000434, + 4.834409064998908, + 4.897851407997223, + 4.978856912999618, + 5.064631416997145, + 5.133863532999385, + 5.191294354997808, + 5.259062032997463, + 5.332868252997287, + 5.408441718998802, + 5.4726324759976706, + 5.537219394998829, + 5.608570741998847, + 5.669731832000252, + 5.732419061998371, + 5.801962972996989, + 5.8550148279973655, + 5.912984778999089, + 5.992941434000386, + 6.084772280999459, + 6.1782765299976745, + 6.2608702029974665, + 6.325599905998388, + 6.400176268998621, + 6.489070305997302, + 6.586602604998916, + 6.663877393999428, + 6.726194987997587, + 6.796522503998858, + 6.862226825996913, + 6.922334309998405, + 6.986704524999368, + 7.0469008539985225, + 7.102543976998277, + 7.160611612998764, + 7.229612101997191, + 7.3241465279970726 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.5472100006591, + "p95": 91.8308469990734, + "max": 97.53229900161386 + } + }, + { + "start": 29430.634668475, + "first": 29430.803083397, + "last": 29437.959127066, + "end": 29437.959850442, + "elapsed_s": 7.32518196700039, + "chunk_offsets_s": [ + 0.16841492200182984, + 0.4170359150011791, + 0.4931982930029335, + 0.5834855560024153, + 0.6517517410029541, + 0.7057715210030437, + 0.7606485850010358, + 0.8218376050026563, + 0.8843610600015381, + 0.965481589002593, + 1.0361139670021657, + 1.121711851003056, + 1.2106826359995466, + 1.2876080490023014, + 1.3499381430010544, + 1.4154825220030034, + 1.4914073150030163, + 1.5683483220018388, + 1.6330986860011762, + 1.7103095350030344, + 1.7803740080016723, + 1.844764262001263, + 1.9311908590025268, + 2.023969945999852, + 2.1086433000018587, + 2.1862491610008874, + 2.2551208570002927, + 2.3149973150029837, + 2.370414274002542, + 2.4217186830028368, + 2.4810192930017365, + 2.550057131000358, + 2.6106914910014893, + 2.66527976100042, + 2.7222598180014757, + 2.784291055002541, + 2.8404441340026096, + 2.9036686190011096, + 2.9673759960023744, + 3.0303628809997463, + 3.096866971001873, + 3.1521389330009697, + 3.218081474002247, + 3.2861432270001387, + 3.340178492002451, + 3.398824464002246, + 3.4599273990024813, + 3.5181617110029038, + 3.583213619000162, + 3.6583672400010983, + 3.7427317190013127, + 3.7993971899995813, + 3.8582801060001657, + 3.916762815002585, + 3.9693004900000233, + 4.022327583999868, + 4.080414404001203, + 4.138836402002198, + 4.203078236001602, + 4.275966885001253, + 4.33244822800043, + 4.398053382999933, + 4.4843268069998885, + 4.5692780730023514, + 4.661777416000405, + 4.750703792000422, + 4.834696387002623, + 4.897997218002274, + 4.979517828000098, + 5.064682145002735, + 5.134515860001557, + 5.19184565100295, + 5.259415296000952, + 5.333165459000156, + 5.408991127002082, + 5.472895471000811, + 5.537368626999523, + 5.609063885000069, + 5.670095474000846, + 5.73255762700137, + 5.802441034000367, + 5.855500134002796, + 5.913321760999679, + 5.993760591001774, + 6.085487240001385, + 6.178420117001224, + 6.261223482000787, + 6.326381462000427, + 6.400779096002225, + 6.4898144530016, + 6.586420578001707, + 6.6642324490021565, + 6.726586380002118, + 6.797088258001168, + 6.86259720200178, + 6.922979569000745, + 6.987233414001821, + 7.04740266600129, + 7.102788260002853, + 7.1608690900029615, + 7.229995337002038, + 7.324458590999711 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.54437900194898, + "p95": 92.49934299805318, + "max": 248.62099299934926 + } + }, + { + "start": 29430.634765139, + "first": 29430.902646999, + "last": 29437.959076798, + "end": 29437.959718015, + "elapsed_s": 7.324952876002499, + "chunk_offsets_s": [ + 0.2678818599997612, + 0.4172588170004019, + 0.49329569400288165, + 0.583073724003043, + 0.6518380750021606, + 0.7059898170009546, + 0.760346949002269, + 0.8223178110019944, + 0.8845483610020892, + 0.9654083720015478, + 1.0359018780000042, + 1.1216813040009583, + 1.2104071320027288, + 1.287760100000014, + 1.3500238050000917, + 1.4156581800016284, + 1.4914023900018947, + 1.5680661959995632, + 1.633101933002763, + 1.71002872200188, + 1.780215353002859, + 1.8447211120001157, + 1.9314712590021372, + 2.0236849100001564, + 2.1080962580017513, + 2.186263316001714, + 2.25477678900279, + 2.3151360920019215, + 2.3700084869997227, + 2.4215678330001538, + 2.4810303779995593, + 2.550218522999785, + 2.610558762000437, + 2.665278930002387, + 2.7221263050014386, + 2.784098014002666, + 2.8404648150026333, + 2.9034203520022857, + 2.9671149509995303, + 3.030430614002398, + 3.0969529540016083, + 3.152122061001137, + 3.217349162001483, + 3.286160102001304, + 3.3399463950008794, + 3.398665633001656, + 3.4596411630009243, + 3.517990501000895, + 3.5831661350021022, + 3.6582073060017137, + 3.7425743429994327, + 3.7992586870022933, + 3.85827534100099, + 3.916583496000385, + 3.9695179560003453, + 4.021885587000725, + 4.08042390600167, + 4.138788454001769, + 4.203180036001868, + 4.275926950002031, + 4.3322872380013, + 4.3981118080009765, + 4.484144817000924, + 4.5692598420027934, + 4.661622982999688, + 4.750538371001312, + 4.834660082000482, + 4.897946871002205, + 4.9794964300017455, + 5.064857260000281, + 5.134017948999826, + 5.191469867000706, + 5.259608164000383, + 5.3330006940013845, + 5.408834215002571, + 5.473016351999831, + 5.53746159900038, + 5.608851043001778, + 5.670040298002277, + 5.7327001320009, + 5.802272847002314, + 5.855449372000294, + 5.913455261001218, + 5.993615914001566, + 6.085177637000015, + 6.178541861001577, + 6.260929025000223, + 6.325995105002221, + 6.4006294930004515, + 6.489477803999762, + 6.586043674000393, + 6.664038368002366, + 6.7264365530027135, + 6.79704841900093, + 6.862015323000378, + 6.922790574000828, + 6.987179228999594, + 7.047351200002595, + 7.102846093999688, + 7.160911730999942, + 7.2301249990014185, + 7.324311659001978 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.22710100034601, + "p95": 92.21365099801915, + "max": 149.3769570006407 + } + }, + { + "start": 29430.634590442, + "first": 29430.719161307, + "last": 29437.959186722, + "end": 29437.959323084, + "elapsed_s": 7.3247326419987075, + "chunk_offsets_s": [ + 0.08457086499765865, + 0.41686499299976276, + 0.49372238499927334, + 0.5830328759984695, + 0.6516327339995769, + 0.7055800309972255, + 0.7602372009969258, + 0.8221629299987399, + 0.8845429549983237, + 0.9649036159971729, + 1.0355724109977018, + 1.1210061210003914, + 1.2103646849973302, + 1.28808959899834, + 1.3503089859987085, + 1.4153022119971865, + 1.4919832109990239, + 1.5679389199976868, + 1.6337317429970426, + 1.7106794819992501, + 1.7798077089973958, + 1.8449613349985157, + 1.931160247997468, + 2.0233993289984937, + 2.1090621869989263, + 2.186953612999787, + 2.2556445819973305, + 2.3149477029983245, + 2.370072124998842, + 2.421939202999056, + 2.4808899219970044, + 2.5502339399972698, + 2.6104186099983053, + 2.6651848549990973, + 2.7221098949994484, + 2.7845571569996537, + 2.840417408999201, + 2.903444772997318, + 2.9669542150004418, + 3.0305287819974183, + 3.0967242769984296, + 3.1527589079996687, + 3.217905992998567, + 3.286424793997867, + 3.33998217199769, + 3.398687142998824, + 3.4599411859999236, + 3.5180265339986363, + 3.583182719998149, + 3.658210647998203, + 3.742514493998897, + 3.799179613997694, + 3.857886552999844, + 3.917080879000423, + 3.969132925998565, + 4.022546888998477, + 4.0802105349976046, + 4.139194849998603, + 4.2034370039982605, + 4.275968132998969, + 4.332676775997243, + 4.3983535669976845, + 4.484169381998072, + 4.569258848998288, + 4.66220732399961, + 4.75048950999917, + 4.83465603500008, + 4.898333754998021, + 4.9799960529999225, + 5.064976235997165, + 5.134118371999648, + 5.192040148998785, + 5.259334719998151, + 5.332942140998057, + 5.408809951997682, + 5.472769937998237, + 5.5373070200002985, + 5.609350569997332, + 5.670014018996881, + 5.733213284998783, + 5.802258195999457, + 5.855163996999181, + 5.912954355997499, + 5.993145125998126, + 6.08560911099994, + 6.179193487998418, + 6.2615334059992165, + 6.326702086000296, + 6.400930091996997, + 6.489970485999947, + 6.586102752997249, + 6.664004203998047, + 6.726509610998619, + 6.7969605489997775, + 6.862796737997996, + 6.922488788997725, + 6.987475060999714, + 7.047660546999396, + 7.102937799998472, + 7.16121197800021, + 7.229996583999309, + 7.324596279999241 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.67679100044188, + "p95": 92.46398500181385, + "max": 332.2941280021041 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 141.83920314353915, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.276464834998478, + "end_to_end_tps": 140.72767796179616, + "timing_valid": true, + "steady_tps": 145.96661256173897, + "per_stream_tps": [ + 35.9, + 36.3, + 36.7, + 35.5 + ], + "ttft_s": [ + 0.17, + 0.25, + 0.33, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16631761399912648, + "decode_tps": 35.86882002981445, + "prefill_tps": null, + "itl_mean_ms": 27.87936707058643, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.4425489999121, + "round_gap_p50_ms": 66.15195699851029, + "round_gap_p95_ms": 92.2428209996724, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.24720496599911712, + "decode_tps": 36.28676978741812, + "prefill_tps": null, + "itl_mean_ms": 27.558253486281235, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.43443899927661, + "round_gap_p50_ms": 65.84209500215366, + "round_gap_p95_ms": 92.3250449995976, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33040026199887507, + "decode_tps": 36.714517660748086, + "prefill_tps": null, + "itl_mean_ms": 27.237182011766194, + "itl_p50_ms": 0.0, + "itl_p95_ms": 85.99600500019733, + "round_gap_p50_ms": 66.4182870023069, + "round_gap_p95_ms": 91.18181500161882, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08516935799707426, + "decode_tps": 35.46278188210344, + "prefill_tps": null, + "itl_mean_ms": 28.19857740784452, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.34562699808157, + "round_gap_p50_ms": 66.26066899843863, + "round_gap_p95_ms": 92.71466700010933, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.718344994119594, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.20163500017952, + "round_gap_p50_ms": 66.15195699851029, + "round_gap_p95_ms": 92.38129000004847, + "round_gap_mean_ms": 69.98196013366828, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29437.961099032, + "first": 29438.127416646, + "last": 29445.236655249, + "end": 29445.237054356, + "elapsed_s": 7.275955323999369, + "chunk_offsets_s": [ + 0.16631761399912648, + 0.3988583270001982, + 0.4767121680015407, + 0.5669224929988559, + 0.6339210750011262, + 0.6887435499993444, + 0.7417856140018557, + 0.804552697998588, + 0.8721099760005018, + 0.955937922000885, + 1.0246435299995937, + 1.110668937999435, + 1.1911897669997416, + 1.2675455489988963, + 1.3294299640001555, + 1.3935216149984626, + 1.4687237270009064, + 1.5460736730019562, + 1.6094265450010425, + 1.6853477789991302, + 1.7562865619984223, + 1.8195871989992156, + 1.914339225000731, + 2.0090389079996385, + 2.094264272000146, + 2.1684109429988894, + 2.2319435230019735, + 2.291030011001567, + 2.343859542001155, + 2.395975308998459, + 2.4530323459985084, + 2.522501674000523, + 2.58082275400011, + 2.6343936220000614, + 2.6897230419999687, + 2.7514841000011074, + 2.8073212380004406, + 2.8750221859991143, + 2.942009958998824, + 3.00546425200082, + 3.071710942000209, + 3.1283942359987122, + 3.1875610429997323, + 3.251426515998901, + 3.305300506999629, + 3.3619632010013447, + 3.422538874001475, + 3.4809676200020476, + 3.547503236000921, + 3.6225119279988576, + 3.7050637770007597, + 3.76070874600191, + 3.819583237000188, + 3.8858048940019216, + 3.9401511549986026, + 3.9957041230009054, + 4.054276121001749, + 4.111723220001295, + 4.169597255000554, + 4.237478453000222, + 4.293787089998659, + 4.35785584399855, + 4.444944505001331, + 4.529651133001607, + 4.62303748100021, + 4.712089606000518, + 4.795196514001873, + 4.861348471000383, + 4.947550106000563, + 5.033992655000475, + 5.102983609001967, + 5.154596444001072, + 5.218432584999391, + 5.290756768001302, + 5.365857854998467, + 5.429573338999035, + 5.493725274998724, + 5.565681795000273, + 5.625157112001034, + 5.688122748000751, + 5.75571164700159, + 5.808236937999027, + 5.8734615729990765, + 5.956613335001748, + 6.04767713199908, + 6.139068848999159, + 6.215340953000123, + 6.277327578998666, + 6.351211055000022, + 6.439950265001244, + 6.538113842001621, + 6.617411250001169, + 6.680259617001866, + 6.749846976999834, + 6.813846139000816, + 6.877624126998853, + 6.947650738999073, + 7.007713668001088, + 7.064337837000494, + 7.122046267999394, + 7.183313395998994, + 7.275556216998666 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.15195699851029, + "p95": 92.2428209996724, + "max": 232.5407130010717 + } + }, + { + "start": 29437.961958386, + "first": 29438.209163352, + "last": 29445.236517991, + "end": 29445.236867469, + "elapsed_s": 7.274909083000239, + "chunk_offsets_s": [ + 0.24720496599911712, + 0.39781290300015826, + 0.475538939001126, + 0.5658916549982678, + 0.6334004499985895, + 0.6874456349978573, + 0.7410727130009036, + 0.8039114809980674, + 0.8713485349981056, + 0.9549119469993457, + 1.0241100300008839, + 1.1100047389991232, + 1.190117313999508, + 1.2662824670005648, + 1.3283778229997552, + 1.3928858360013692, + 1.4677844680009002, + 1.5451637789992674, + 1.608141071999853, + 1.6844143899979827, + 1.7551031609982601, + 1.8185046859980503, + 1.9133110420007142, + 2.00807514799817, + 2.0929556080009206, + 2.16746127299848, + 2.231348191999132, + 2.289854254999227, + 2.34276305000094, + 2.395066326000233, + 2.45222713000112, + 2.521845694998774, + 2.579900128999725, + 2.633350244999747, + 2.6889461669998127, + 2.7515476279986615, + 2.806311927000934, + 2.8740868940003566, + 2.9413025290014048, + 3.0043318059979356, + 3.0710171860009723, + 3.127311849999387, + 3.186160719000327, + 3.250080010999227, + 3.304140407999512, + 3.3613300930010155, + 3.4216131620014494, + 3.479953384001419, + 3.5464091430003464, + 3.6213390989978507, + 3.704436075000558, + 3.7595874959988578, + 3.8186107910005376, + 3.884999550999055, + 3.9397259010002017, + 3.9944516440009465, + 4.052391189001355, + 4.110930816001201, + 4.169053615998564, + 4.236281248999148, + 4.293150768000487, + 4.3571670620003715, + 4.443825318001473, + 4.529580106998765, + 4.621961396998813, + 4.71087012299904, + 4.794506884998555, + 4.860348980000708, + 4.9465958299988415, + 5.033030268998118, + 5.101972218999435, + 5.153911197998241, + 5.218022748998919, + 5.2896425379985885, + 5.364723970000341, + 5.428513679999014, + 5.492769575001148, + 5.564647118000721, + 5.62425163199805, + 5.686955578999914, + 5.75476786299987, + 5.807179487001122, + 5.872692422999535, + 5.955362935001176, + 6.046990253998956, + 6.138148367001122, + 6.213966121998965, + 6.276317307998397, + 6.350005101998249, + 6.438667924998299, + 6.537076622000313, + 6.616307783999218, + 6.679277111001284, + 6.749146408998058, + 6.813411768998776, + 6.877063375999569, + 6.9463368600008835, + 7.00692615799926, + 7.063527391997923, + 7.121479055000236, + 7.182234560001234, + 7.274559605000832 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.84209500215366, + "p95": 92.3250449995976, + "max": 150.60793700104114 + } + }, + { + "start": 29437.961533611, + "first": 29438.291933873, + "last": 29445.237415286, + "end": 29445.237469009, + "elapsed_s": 7.275935398000001, + "chunk_offsets_s": [ + 0.33040026199887507, + 0.3983756870002253, + 0.47619170300094993, + 0.5664249309993465, + 0.633636212998681, + 0.688049698001123, + 0.741843957999663, + 0.8039329130006081, + 0.8718387080007233, + 0.9554237429983914, + 1.0243859680012974, + 1.1097792440014018, + 1.1908662479982013, + 1.2664635629989789, + 1.3288977919983154, + 1.3929432529985206, + 1.4684132100010174, + 1.5458682989992667, + 1.6088765720014635, + 1.6849892179998278, + 1.7557770210005401, + 1.8190579699985392, + 1.9135553449996223, + 2.0086978920007823, + 2.0939485730013985, + 2.167362336000224, + 2.2311786909995135, + 2.2905086669998127, + 2.343728874999215, + 2.395258681000996, + 2.452703069000563, + 2.5221567709995725, + 2.5800216479983646, + 2.6334464790015772, + 2.6895324299985077, + 2.7511218930012546, + 2.8071273630012, + 2.8743881340014923, + 2.9412389700009953, + 3.004637059999368, + 3.071055347001675, + 3.128233660998376, + 3.1860885069982032, + 3.250934904001042, + 3.3052432180011238, + 3.3619654729991453, + 3.4221638870003517, + 3.4800936950014147, + 3.546661011001561, + 3.62215584599835, + 3.704305965999083, + 3.7604520009990665, + 3.8188192199995683, + 3.885251913998218, + 3.9400427990003664, + 3.994964481000352, + 4.052963473001, + 4.111743869001657, + 4.169415616001061, + 4.236998003998451, + 4.293142817998159, + 4.357261664001271, + 4.4438950330004445, + 4.528740085999743, + 4.622278742001072, + 4.711101421999047, + 4.794390530001692, + 4.861079026999505, + 4.947250945999258, + 5.033246950999455, + 5.102301544000511, + 5.153902559999551, + 5.218284260998189, + 5.290175574999012, + 5.365248704998521, + 5.428820214001462, + 5.493404714998178, + 5.564950101001159, + 5.6248593109994545, + 5.687242520998552, + 5.755330118998245, + 5.80769609699928, + 5.8731824360002065, + 5.956028446998971, + 6.04721026200059, + 6.1382307009989745, + 6.214867908998713, + 6.276660323001124, + 6.35035889700157, + 6.4393066800003, + 6.537824309998541, + 6.616649488001713, + 6.679588363000221, + 6.74982405999981, + 6.81370728299953, + 6.877389533998212, + 6.947032536001643, + 7.00719758500054, + 7.063282333998359, + 7.121213110000099, + 7.182519422000041, + 7.275881674999255 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.4182870023069, + "p95": 91.18181500161882, + "max": 98.51762999824132 + } + }, + { + "start": 29437.961004174, + "first": 29438.046173532, + "last": 29445.236810771, + "end": 29445.23727774, + "elapsed_s": 7.2762735659998725, + "chunk_offsets_s": [ + 0.08516935799707426, + 0.3985227489974932, + 0.47633537399815395, + 0.5667568469980324, + 0.6340694319987961, + 0.6887834379995184, + 0.7421391369971388, + 0.8042062439999427, + 0.8725677299989911, + 0.9560837680000986, + 1.024980563997815, + 1.1101478969976597, + 1.1911520539979392, + 1.2667862339985732, + 1.3291012310000951, + 1.3932132089976221, + 1.4685851260001073, + 1.5459077059967967, + 1.6092521979990124, + 1.6852381259996037, + 1.7557937019992096, + 1.8193358589996933, + 1.9147315489972243, + 2.0088085039969883, + 2.094107852997695, + 2.168167519997951, + 2.2312160349974874, + 2.290568450000137, + 2.3440297139968607, + 2.3958495979968575, + 2.452803715998016, + 2.522425798997574, + 2.580330356999184, + 2.634135229000094, + 2.689744632996735, + 2.751474791999499, + 2.8074620939987653, + 2.874815867999132, + 2.941819185998611, + 3.005378876998293, + 3.0716395459967316, + 3.1284104049991583, + 3.1863032359979115, + 3.251248697997653, + 3.3052477429991995, + 3.36253079999733, + 3.4224000799986243, + 3.4805354549971526, + 3.547098195998842, + 3.6225001409984543, + 3.7049478230001114, + 3.7608997899988026, + 3.8192789889981213, + 3.8856040129976464, + 3.940443231000245, + 3.995859644997836, + 4.0541646129968285, + 4.11211456899764, + 4.169391335999535, + 4.237905787998898, + 4.293726553998567, + 4.3577503169981355, + 4.444128570998146, + 4.529221797998616, + 4.6229520099986985, + 4.711290609997377, + 4.7950355209977715, + 4.861546087999159, + 4.94789171499724, + 5.033518632997584, + 5.102676065998821, + 5.154102766999131, + 5.218286225997872, + 5.290457813000103, + 5.365358179999021, + 5.429169785998965, + 5.493565016997309, + 5.565236065998761, + 5.6249914489999355, + 5.687553627998568, + 5.755973942999844, + 5.808289844997489, + 5.873747458997968, + 5.956637356997817, + 6.047839771999861, + 6.138809014999424, + 6.215054632997635, + 6.277074781999545, + 6.350745621999522, + 6.439332698999351, + 6.5377400209981715, + 6.617047648996959, + 6.679919253998378, + 6.75031173699972, + 6.81399105699893, + 6.878309472998808, + 6.9478819589967316, + 7.007545205997303, + 7.063875122999889, + 7.121801067998604, + 7.1830919299973175, + 7.275806596997427 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.26066899843863, + "p95": 92.71466700010933, + "max": 313.3533910004189 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.7, + "per_stream_tps": 33.7, + "ttft_s_max": 0.42, + "request": { + "ttft_s": 0.254, + "decode_tps": 33.658, + "prefill_tps": null, + "itl_mean_ms": 29.71, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.183, + "round_gap_p50_ms": 68.928, + "round_gap_p95_ms": 98.608, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 29.71, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.94, + "round_gap_p50_ms": 68.928, + "round_gap_p95_ms": 98.698, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 163.562654997134, + "makespan_s": 7.8257472650002455, + "steady_tps": 170.9, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.7043216352773, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.8257472650002455, + "end_to_end_tps": 163.562654997134, + "timing_valid": true, + "steady_tps": 170.89944849413905, + "per_stream_tps": [ + 33.7, + 32.9, + 34.4, + 34.0, + 33.3 + ], + "ttft_s": [ + 0.25, + 0.08, + 0.41, + 0.33, + 0.16 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.24839688200154342, + "decode_tps": 33.65819837021424, + "prefill_tps": null, + "itl_mean_ms": 29.71044347058541, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.52373200069997, + "round_gap_p50_ms": 68.50303999817697, + "round_gap_p95_ms": 98.97041699878173, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08400041299682925, + "decode_tps": 32.944109667590936, + "prefill_tps": null, + "itl_mean_ms": 30.354439992159172, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.73791500143125, + "round_gap_p50_ms": 68.41829000040889, + "round_gap_p95_ms": 99.53616300117574, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.4149121650007146, + "decode_tps": 34.41325405280889, + "prefill_tps": null, + "itl_mean_ms": 29.0585714000033, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.06988699932117, + "round_gap_p50_ms": 68.1521389997215, + "round_gap_p95_ms": 98.52971000145772, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33154152000133763, + "decode_tps": 34.029647132229954, + "prefill_tps": null, + "itl_mean_ms": 29.38614074116819, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.9971479997912, + "round_gap_p50_ms": 68.08963599905837, + "round_gap_p95_ms": 99.90863500206615, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16464497899869457, + "decode_tps": 33.28709715072029, + "prefill_tps": null, + "itl_mean_ms": 30.041670364709503, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.61965100085945, + "round_gap_p50_ms": 68.37884999913513, + "round_gap_p95_ms": 99.33357900081319, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.710253193725116, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.88950399766327, + "round_gap_p50_ms": 68.39622000188683, + "round_gap_p95_ms": 99.54839100100799, + "round_gap_mean_ms": 75.01103529108816, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29445.239258321, + "first": 29445.487655203, + "last": 29453.063818288, + "end": 29453.064498616, + "elapsed_s": 7.825240295002004, + "chunk_offsets_s": [ + 0.24839688200154342, + 0.48717749700153945, + 0.5736218150013883, + 0.6735640280021471, + 0.7464155289999326, + 0.803133859000809, + 0.8597117320023244, + 0.9260949560011795, + 0.9929666290008754, + 1.0767994369998632, + 1.1513454250016366, + 1.2438691570023366, + 1.3281489410001086, + 1.4065920720022405, + 1.4719179029998486, + 1.5381561130016053, + 1.6259789089999686, + 1.7104559979998157, + 1.7785185749999073, + 1.859920853003132, + 1.9346836840013566, + 2.001111142002628, + 2.091954777002684, + 2.1909251940014656, + 2.2793282170023303, + 2.3555154980022053, + 2.423484255999938, + 2.4854779160014004, + 2.5412041590025183, + 2.6022613720015215, + 2.6665948270019726, + 2.741256864999741, + 2.8059183190016483, + 2.8630917910013522, + 2.9220116840006085, + 2.988210601000901, + 3.047821631000261, + 3.113864545000979, + 3.183545571002469, + 3.252048611000646, + 3.3231552240031306, + 3.382551730002888, + 3.4427400860004127, + 3.513418131002254, + 3.5726404010019905, + 3.63959367100324, + 3.7053350630012574, + 3.766623810002784, + 3.833850317001634, + 3.912529903001996, + 4.005623394001304, + 4.065581173003011, + 4.126351867002086, + 4.188527499001793, + 4.244014513002185, + 4.2986210380004195, + 4.361405313000432, + 4.427298742000858, + 4.487282382000558, + 4.563601380003092, + 4.630766831000074, + 4.699488027003099, + 4.791377531000762, + 4.879365580000012, + 4.983020351002779, + 5.076535812000657, + 5.1692386790018645, + 5.23736960899987, + 5.325659429003281, + 5.414672678001807, + 5.490109015001508, + 5.5440866660028405, + 5.61958244500056, + 5.700415923001856, + 5.780083919002209, + 5.848087801001384, + 5.915369405000092, + 5.990828768000938, + 6.0530932060028135, + 6.119207313000516, + 6.194047988999955, + 6.250461425002868, + 6.311860519999755, + 6.398341159001575, + 6.496518202002335, + 6.595480245003273, + 6.68731537700296, + 6.754739826999867, + 6.833208413001557, + 6.925948664000316, + 7.030590319001931, + 7.116242896001495, + 7.184110420999787, + 7.258509454000887, + 7.328603585003293, + 7.3911555129998305, + 7.461850163002964, + 7.526373260003311, + 7.586916264001047, + 7.651990670001396, + 7.720386890003283, + 7.824559967000823 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.50303999817697, + "p95": 98.97041699878173, + "max": 238.78061499999603 + } + }, + { + "start": 29445.239323352, + "first": 29445.323323765, + "last": 29453.063705963, + "end": 29453.064074588, + "elapsed_s": 7.824751235999429, + "chunk_offsets_s": [ + 0.08400041299682925, + 0.4873875089979265, + 0.5737644789987826, + 0.6733006419999583, + 0.7467729879972467, + 0.803244934999384, + 0.8598535639976035, + 0.925799199998437, + 0.9924377799979993, + 1.0763530029980757, + 1.1511425439966843, + 1.243616583997209, + 1.3277062179986387, + 1.40639072599879, + 1.471908130999509, + 1.5379669059984735, + 1.625841617998958, + 1.709855080996931, + 1.7779286449986103, + 1.8600152469989553, + 1.9345495279994793, + 2.0013400119969447, + 2.0912877729970205, + 2.1901165029994445, + 2.2790805219992762, + 2.355337999997573, + 2.4229564469969773, + 2.484985819999565, + 2.5410876919995644, + 2.602355062997958, + 2.666079363996687, + 2.7408428819981054, + 2.8057307439994474, + 2.862458613999479, + 2.923048654000013, + 2.988385747998109, + 3.047666730999481, + 3.1136185449977347, + 3.1837745849989005, + 3.251425061996997, + 3.322445836998668, + 3.382030302000203, + 3.44283496099888, + 3.5131537219967868, + 3.5725029179993726, + 3.639301991999673, + 3.7056711389996053, + 3.7667640909967304, + 3.8347692059978726, + 3.9128303289980977, + 4.005391740996856, + 4.065744084997277, + 4.126121523997426, + 4.188999698999396, + 4.243676132999099, + 4.298391623997304, + 4.360889913998108, + 4.427810892997513, + 4.487386436998349, + 4.5630879959971935, + 4.630912661999901, + 4.699798033998377, + 4.791535948999808, + 4.878473511998891, + 4.98257673199987, + 5.076538146997336, + 5.168984265998006, + 5.237402555998415, + 5.325930234997941, + 5.414491420997365, + 5.490188887997647, + 5.543700719998014, + 5.619344489998184, + 5.699893501998304, + 5.7794265749980696, + 5.847264025996992, + 5.9148460409996915, + 5.990334351998172, + 6.052815777999058, + 6.118909475997498, + 6.193837717997667, + 6.2500029479997465, + 6.311374932996841, + 6.398425061997841, + 6.495660040000075, + 6.595602517998486, + 6.686917691997223, + 6.754251599999407, + 6.833758278997266, + 6.925432528998499, + 7.0303570500000205, + 7.1157026089967985, + 7.183915583998896, + 7.259338474999822, + 7.328808272999595, + 7.390942907997669, + 7.46200878899981, + 7.526192194996838, + 7.586705608999182, + 7.651854738000111, + 7.719758589999401, + 7.824382610997418 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.41829000040889, + "p95": 99.53616300117574, + "max": 403.38709600109723 + } + }, + { + "start": 29445.239544849, + "first": 29445.654457014, + "last": 29453.064392721, + "end": 29453.06494489, + "elapsed_s": 7.825400041001558, + "chunk_offsets_s": [ + 0.4149121650007146, + 0.4865390750019287, + 0.5731228750009905, + 0.6736422360008874, + 0.7461865790028241, + 0.8032261100015603, + 0.8593330799994874, + 0.9256837089997134, + 0.9923744450024969, + 1.0766380109998863, + 1.1510189610016823, + 1.243282891002309, + 1.3276011720008682, + 1.4058913860026223, + 1.4712776740016125, + 1.5376767640009348, + 1.6257749880023766, + 1.7099465009996493, + 1.7778852390001703, + 1.8591374790012196, + 1.9348150090008858, + 2.0010186020008405, + 2.0915355649995035, + 2.1900652750009613, + 2.2789873740002804, + 2.3553191909995803, + 2.4229990060011914, + 2.485100367000996, + 2.5408129040006315, + 2.602080387001479, + 2.666043442000955, + 2.7408829470005003, + 2.805966140000237, + 2.862642191001214, + 2.9221268870023778, + 2.987773699001991, + 3.0478192160007893, + 3.113995853000233, + 3.183214549000695, + 3.251055767002981, + 3.322522079000919, + 3.381916826001543, + 3.441670617001364, + 3.513038951001363, + 3.5725695800028916, + 3.6393638730005478, + 3.7049570820017834, + 3.7663841110006615, + 3.8336034220010333, + 3.912339577000239, + 4.005502354000782, + 4.065367289000278, + 4.1260984300024575, + 4.188040330001968, + 4.243799157000467, + 4.298297069002729, + 4.3609047410027415, + 4.4274104420001095, + 4.487038254002982, + 4.563413054002012, + 4.630646494002576, + 4.699656426000729, + 4.79072631300005, + 4.878620239000156, + 4.982819310000195, + 5.075966819000314, + 5.168990840000333, + 5.236880055999791, + 5.325571063000098, + 5.414771504001692, + 5.489899225001864, + 5.543566837000981, + 5.618579332000081, + 5.699981916000979, + 5.780006684999535, + 5.847402133000287, + 5.914861827001005, + 5.990719099001581, + 6.053001722000772, + 6.118868165001913, + 6.1937144069997885, + 6.2503998820029665, + 6.31175182700099, + 6.398128906002967, + 6.4960297210018325, + 6.5955781120028405, + 6.686623343000974, + 6.75458115299989, + 6.833050282999466, + 6.92571144500107, + 7.030349901000591, + 7.116094650002196, + 7.183921343003021, + 7.2582682040010695, + 7.328360464001889, + 7.391114762001962, + 7.461927477001154, + 7.526044573001855, + 7.587100800999906, + 7.6519037280013436, + 7.720055867001065, + 7.8248478720015555 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.1521389997215, + "p95": 98.52971000145772, + "max": 104.79200500049046 + } + }, + { + "start": 29445.239414148, + "first": 29445.570955668, + "last": 29453.064421557, + "end": 29453.064618873, + "elapsed_s": 7.825204724998912, + "chunk_offsets_s": [ + 0.33154152000133763, + 0.48635159699915675, + 0.5728937489984673, + 0.6738641820011253, + 0.7461988460017892, + 0.8033092459991167, + 0.8594043170014629, + 0.9257759610009089, + 0.9924539499988896, + 1.076378305999242, + 1.1511097570000857, + 1.2433611000014935, + 1.3278530889983813, + 1.4057955040007073, + 1.471294211998611, + 1.538162717999512, + 1.6255674060012097, + 1.7104966699989745, + 1.7780706539997482, + 1.8593610880016058, + 1.9350894130002416, + 2.0013027310014877, + 2.091838614000153, + 2.1903282919993217, + 2.2793826759989315, + 2.3554183839987672, + 2.4232725469992147, + 2.4853650000004563, + 2.540826371001458, + 2.6023278259999643, + 2.6665250779988128, + 2.7410602060008387, + 2.805856342998595, + 2.8629738050003652, + 2.9220497970018187, + 2.988092791001691, + 3.0477559130013105, + 3.1136381050018826, + 3.18352530699849, + 3.251426997998351, + 3.3226192420006555, + 3.3822849380012485, + 3.44165812600113, + 3.5133009609999135, + 3.5725350180000532, + 3.6393974599996, + 3.7051326619985048, + 3.766731687999709, + 3.834125937999488, + 3.912523538001551, + 4.005563546001213, + 4.065387153001211, + 4.126158022998425, + 4.188448601998971, + 4.24376592200133, + 4.2985039800005325, + 4.36129004599934, + 4.427373529000761, + 4.487708232001751, + 4.563698031000968, + 4.631031720000465, + 4.699825288000284, + 4.791637605998403, + 4.8790061049985525, + 4.983300113999576, + 5.076301823999529, + 5.169190553999215, + 5.2371740540002065, + 5.326291950001178, + 5.4151124589989195, + 5.490268872999877, + 5.5438954290002584, + 5.619036290001532, + 5.700194042001385, + 5.779562394000095, + 5.847459325999807, + 5.9149286649990245, + 5.990720969999529, + 6.052841208998871, + 6.119117843998538, + 6.1939914989998215, + 6.2500823090013, + 6.311964447999344, + 6.398041484000714, + 6.4957461029989645, + 6.595654738001031, + 6.686916828999529, + 6.75467826699969, + 6.833466488998965, + 6.9254636369987566, + 7.030543873999704, + 7.1161803280010645, + 7.183996689000196, + 7.258536563000234, + 7.328337177001231, + 7.391467583998747, + 7.461998585000401, + 7.525989218000177, + 7.586844116001885, + 7.6519930210015445, + 7.720082657000603, + 7.825007408999227 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.08963599905837, + "p95": 99.90863500206615, + "max": 154.81007699781912 + } + }, + { + "start": 29445.239197625, + "first": 29445.403842604, + "last": 29453.064468547, + "end": 29453.064737355, + "elapsed_s": 7.825539729998127, + "chunk_offsets_s": [ + 0.16464497899869457, + 0.4871189119985502, + 0.5732131270015088, + 0.6735112800015486, + 0.7468187790000229, + 0.8032966270002362, + 0.8600821149993862, + 0.9260889750003116, + 0.9928440209987457, + 1.0770655240012275, + 1.1512289109996345, + 1.2440967000002274, + 1.3281132269985392, + 1.4063323180016596, + 1.4718882349989144, + 1.5379697380012658, + 1.6258660409985168, + 1.7101176180003677, + 1.778173845999845, + 1.859777164998377, + 1.935057250000682, + 2.0009148030003416, + 2.0917130630004976, + 2.190915422001126, + 2.2796672039985424, + 2.355902994000644, + 2.423189747998549, + 2.485263998998562, + 2.541413900999032, + 2.602821248998225, + 2.6668509660012205, + 2.7415062069994747, + 2.8064585069987515, + 2.862799537000683, + 2.922617669999454, + 2.988623158998962, + 3.0482823780002946, + 3.114568668999709, + 3.18406990999938, + 3.2520015410009364, + 3.323025450001296, + 3.382814906999556, + 3.442577429999801, + 3.5136724629992386, + 3.5728271750012937, + 3.6400059900006454, + 3.7056802549996064, + 3.7671736589982174, + 3.8342853629983438, + 3.91314668600171, + 4.006102423001721, + 4.065927389998251, + 4.126640586000576, + 4.18895814699863, + 4.244358986001316, + 4.298924982998869, + 4.361651712999446, + 4.427644142000645, + 4.487673829000414, + 4.563268437999795, + 4.6309209309984, + 4.699615192999772, + 4.7911458290000155, + 4.879363644999103, + 4.9833578190009575, + 5.076960509999481, + 5.16958016100034, + 5.237746597998921, + 5.326440500000899, + 5.415489686998626, + 5.490434264000214, + 5.544238157999644, + 5.6188512580010865, + 5.700155239999731, + 5.779865571999835, + 5.84790473600151, + 5.915490653998859, + 5.990528132999316, + 6.0532417239992355, + 6.1195362559992645, + 6.194164102998911, + 6.250591454001551, + 6.311654231998546, + 6.398734428999887, + 6.496316664000915, + 6.595650243001728, + 6.6875302369990095, + 6.755221207000432, + 6.833326749998378, + 6.926299407998158, + 7.03089061799983, + 7.116150018999178, + 7.184456205999595, + 7.259413644998858, + 7.329149820001476, + 7.391644586998154, + 7.4624269460000505, + 7.526634917998308, + 7.587220465000428, + 7.652121450999402, + 7.720500300998538, + 7.825270921999618 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.37884999913513, + "p95": 99.33357900081319, + "max": 322.4739329998556 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.1277168089792, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.854534239999339, + "end_to_end_tps": 162.963195638206, + "timing_valid": true, + "steady_tps": 170.36500934289887, + "per_stream_tps": [ + 34.3, + 32.8, + 33.2, + 33.9, + 33.6 + ], + "ttft_s": [ + 0.42, + 0.09, + 0.17, + 0.34, + 0.25 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.420573568000691, + "decode_tps": 34.305621573852676, + "prefill_tps": null, + "itl_mean_ms": 29.1497414745048, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.44430499893497, + "round_gap_p50_ms": 70.15088199841557, + "round_gap_p95_ms": 97.6892830003635, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08565474900024128, + "decode_tps": 32.82846827205762, + "prefill_tps": null, + "itl_mean_ms": 30.461366388244286, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.35435500158928, + "round_gap_p50_ms": 70.39436700142687, + "round_gap_p95_ms": 97.81099599786103, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.169066609996662, + "decode_tps": 33.18597407377289, + "prefill_tps": null, + "itl_mean_ms": 30.133212235295122, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.03266199983773, + "round_gap_p50_ms": 70.46726699991268, + "round_gap_p95_ms": 98.60843400019803, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3367409269994823, + "decode_tps": 33.924520262864846, + "prefill_tps": null, + "itl_mean_ms": 29.47720387057737, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.32348499790533, + "round_gap_p50_ms": 70.64534199889749, + "round_gap_p95_ms": 98.12972099825856, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2535976670005766, + "decode_tps": 33.55272667002626, + "prefill_tps": null, + "itl_mean_ms": 29.803837101958468, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.23379400282283, + "round_gap_p50_ms": 70.65442599923699, + "round_gap_p95_ms": 98.69917499963776, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.80507221411601, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.23379400282283, + "round_gap_p50_ms": 70.46726699991268, + "round_gap_p95_ms": 98.68873500090558, + "round_gap_mean_ms": 75.25042984752062, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29453.066925945, + "first": 29453.487499513, + "last": 29460.920683589, + "end": 29460.921127322, + "elapsed_s": 7.85420137699839, + "chunk_offsets_s": [ + 0.420573568000691, + 0.4907244499991066, + 0.5756813259977207, + 0.6689814579985978, + 0.7408550100008142, + 0.8067817909977748, + 0.8670231299984152, + 0.9325748570008727, + 0.9976764060011192, + 1.0832093579992943, + 1.1590503900006297, + 1.2486610249980004, + 1.3339887449983507, + 1.4150310240002, + 1.4817900419984653, + 1.5496706099984294, + 1.630860063, + 1.7129040809995786, + 1.7860209869977552, + 1.8691282660001889, + 1.946570330997929, + 2.0132062150005368, + 2.105241859000671, + 2.2039715350001643, + 2.292489965999266, + 2.3702259979982045, + 2.4379273720005585, + 2.4995702460000757, + 2.555614152999624, + 2.609793740997702, + 2.6702730340002745, + 2.745007600999088, + 2.817615542000567, + 2.8760629139978846, + 2.9353694089986675, + 3.0027182810008526, + 3.0628993669997726, + 3.1302480150006886, + 3.2000659959994664, + 3.2682369180001842, + 3.3400626180009567, + 3.398527886998636, + 3.458189269000286, + 3.5323364389987546, + 3.589861287000531, + 3.649351439999009, + 3.714305958001205, + 3.7804340990005585, + 3.8511245909976424, + 3.9299572339987208, + 4.022427773998061, + 4.0811722750004265, + 4.141615862998151, + 4.204188339997927, + 4.258683330997883, + 4.3126150669995695, + 4.372477902998071, + 4.437020195000514, + 4.497793623999314, + 4.574366883000039, + 4.635117616999196, + 4.701682406997861, + 4.800609033998626, + 4.894053337000514, + 4.996410871997796, + 5.090980500001024, + 5.183524637999653, + 5.250642701998004, + 5.338877526999568, + 5.42684778200055, + 5.503103855000518, + 5.558496779998677, + 5.625704264999513, + 5.705191123000986, + 5.789200201001222, + 5.860651951999898, + 5.928095390998351, + 6.007609026000864, + 6.072409595999488, + 6.1402230380008405, + 6.215716957998666, + 6.2709397159997025, + 6.333701633000601, + 6.419794682999054, + 6.517483965999418, + 6.612737837000168, + 6.698401707999437, + 6.769245425999543, + 6.8546871799990186, + 6.9471314849979535, + 7.052332620998641, + 7.136541042000317, + 7.204179282998666, + 7.2765893830001005, + 7.3458780929977365, + 7.407851412001037, + 7.4783790209985455, + 7.543331895998563, + 7.601351474000694, + 7.662827352000022, + 7.729216430998349, + 7.8537576439994154 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.15088199841557, + "p95": 97.6892830003635, + "max": 124.54121300106635 + } + }, + { + "start": 29453.066688277, + "first": 29453.152343026, + "last": 29460.919991455, + "end": 29460.920841256, + "elapsed_s": 7.85415297900181, + "chunk_offsets_s": [ + 0.08565474900024128, + 0.49144500300099025, + 0.5750231340025493, + 0.66910948900113, + 0.7412299690004147, + 0.8073069640013273, + 0.8670076160015014, + 0.9327611520020582, + 0.9982067760029167, + 1.083730756003206, + 1.159049527002935, + 1.2492947960017773, + 1.3339638430006744, + 1.4153930920001585, + 1.4817146000023058, + 1.5502386770021985, + 1.6309056930003862, + 1.71298214700073, + 1.7861093370011076, + 1.869734239000536, + 1.9463023710013658, + 2.0138107490020047, + 2.105373808000877, + 2.2031610650010407, + 2.2930186730009154, + 2.3700651160033885, + 2.4380367220001062, + 2.5000372170034098, + 2.5560651780033368, + 2.6095090840026387, + 2.66975262699998, + 2.744852557003469, + 2.8175969250005437, + 2.8761523510002007, + 2.935178891002579, + 3.0024403250026808, + 3.0624138740022318, + 3.1298766220024845, + 3.2000541600027645, + 3.2681531740017817, + 3.340194998003426, + 3.39750791300321, + 3.4581501160027983, + 3.532436832003441, + 3.590277191000496, + 3.6500100810008007, + 3.7146755430003395, + 3.7807264820003184, + 3.8511208490017452, + 3.930134333000751, + 4.02216282000154, + 4.081205014001171, + 4.141187148001336, + 4.203974231000757, + 4.258520433002559, + 4.31242699500217, + 4.372251767003036, + 4.436682565999945, + 4.498310640003183, + 4.574289457003033, + 4.635103014003107, + 4.701104295003461, + 4.800601277001988, + 4.894241824000346, + 4.99650528300117, + 5.091004890000477, + 5.183404668001458, + 5.250465218003228, + 5.338523599002656, + 5.426812468002026, + 5.502634708002006, + 5.55820268500247, + 5.625432610002463, + 5.7044163970022055, + 5.789923874002852, + 5.8604526840026665, + 5.92861092000021, + 6.007059685001877, + 6.072610431001522, + 6.140215408002405, + 6.215731879001396, + 6.271286926003086, + 6.3336379770007625, + 6.419689827002003, + 6.517500822999864, + 6.612860717003059, + 6.697941725000419, + 6.768991379001818, + 6.854808109001169, + 6.9471624640027585, + 7.0522943470023165, + 7.136644985002931, + 7.203998568002135, + 7.27643984000315, + 7.345534241001587, + 7.407588761001534, + 7.478847736001626, + 7.5435285400017165, + 7.601730783000676, + 7.662895486002526, + 7.729172927000036, + 7.853303178002534 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.39436700142687, + "p95": 97.81099599786103, + "max": 405.790254000749 + } + }, + { + "start": 29453.067037805, + "first": 29453.236104415, + "last": 29460.920073535, + "end": 29460.920277201, + "elapsed_s": 7.853239395997662, + "chunk_offsets_s": [ + 0.169066609996662, + 0.49103033199935453, + 0.5754388609966554, + 0.6689439689980645, + 0.7407847179965756, + 0.806561604997114, + 0.8664381409980706, + 0.9325631179999618, + 0.9976695449986437, + 1.0830470059991057, + 1.1592442999972263, + 1.249130955999135, + 1.3337083749975136, + 1.4146962899976643, + 1.4816277379977691, + 1.5490342489974864, + 1.6306042910000542, + 1.712962282999797, + 1.7852574270000332, + 1.8689046089966723, + 1.9463203489976877, + 2.012737452998408, + 2.1056667039993044, + 2.203089093996823, + 2.292188116000034, + 2.3698710319986276, + 2.437883309998142, + 2.499768457997561, + 2.555367336997733, + 2.6093741609984136, + 2.6696906039978785, + 2.744430304999696, + 2.817390317999525, + 2.875749115999497, + 2.9350513409990526, + 3.0026915249982267, + 3.0622613099985756, + 3.1298894659994403, + 3.2000029489972803, + 3.268080594996718, + 3.340347931996803, + 3.3976164479972795, + 3.458125357999961, + 3.5321543179998116, + 3.5900789159968554, + 3.6497922299968195, + 3.714243230999273, + 3.78047733099811, + 3.850944597998023, + 3.9299946439969062, + 4.022027305996744, + 4.081251124996925, + 4.141180911999982, + 4.20387569399827, + 4.258630039999844, + 4.312423589999526, + 4.372648077998747, + 4.436860945999797, + 4.497900063997804, + 4.574512779996439, + 4.6349395899997035, + 4.701524420997885, + 4.8003216099968995, + 4.894012169999769, + 4.99671280399707, + 5.091351477996795, + 5.183179459996609, + 5.250198185996851, + 5.338531500998215, + 5.4267903009967995, + 5.502554563998274, + 5.558073327996681, + 5.625359534999006, + 5.704725510997378, + 5.789403659997333, + 5.859951342998102, + 5.928157160997216, + 6.007277234999492, + 6.071863423996547, + 6.14034284799709, + 6.2159927879984025, + 6.270157010996627, + 6.33353499399891, + 6.41925359599918, + 6.517862029999378, + 6.612749721, + 6.697725617999822, + 6.768773624997266, + 6.855031399998552, + 6.946939542998734, + 7.051825216996804, + 7.136101771997346, + 7.203765634996671, + 7.27631472199937, + 7.3453983109975525, + 7.407419850998849, + 7.47859839299781, + 7.542832872997678, + 7.60084219899727, + 7.663033449996874, + 7.728628593999019, + 7.853035729996918 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.46726699991268, + "p95": 98.60843400019803, + "max": 321.9637220026925 + } + }, + { + "start": 29453.066814868, + "first": 29453.403555795, + "last": 29460.920242782, + "end": 29460.920698383, + "elapsed_s": 7.853883514999325, + "chunk_offsets_s": [ + 0.3367409269994823, + 0.4909596709985635, + 0.574743306999153, + 0.6694161369996436, + 0.740679126996838, + 0.8070169639977394, + 0.8670541739993496, + 0.9324569819982571, + 0.9977260669984389, + 1.0829925789985282, + 1.159661416997551, + 1.249053000999993, + 1.333980380997673, + 1.41496389799795, + 1.4817660519984202, + 1.5499428399998578, + 1.631048182000086, + 1.7133616309984063, + 1.7856168709986378, + 1.869283773998177, + 1.946718625997164, + 2.013204359998781, + 2.1055278449966863, + 2.2034119599993574, + 2.292494027999055, + 2.370298896999884, + 2.4381550759972015, + 2.499723130997154, + 2.5553490389975195, + 2.6096426799995243, + 2.669863847997476, + 2.7448331879968464, + 2.817675839000003, + 2.876122267000028, + 2.9352297029981855, + 3.0025958649966924, + 3.0626773559997673, + 3.1300376479994156, + 3.200267501997587, + 3.2684268439988955, + 3.3407638179996866, + 3.398096884997358, + 3.4584245859987277, + 3.5324176229987643, + 3.5902165259976755, + 3.649977629997011, + 3.714635398999235, + 3.780645024999103, + 3.8512903669980005, + 3.930482198997197, + 4.022481705000246, + 4.081637086997944, + 4.141271867996693, + 4.204194833997462, + 4.258579034998547, + 4.312793141998554, + 4.372023742998863, + 4.437245948000054, + 4.498305681998318, + 4.574559688997397, + 4.635364832996856, + 4.70190481699683, + 4.800593551997736, + 4.8940539289978915, + 4.997000403996935, + 5.091755672998261, + 5.183363387997815, + 5.25056413999846, + 5.339116650997312, + 5.427111791999778, + 5.50294543599739, + 5.558253338000213, + 5.625539208998816, + 5.704781919997913, + 5.7894170619983925, + 5.860068688998581, + 5.9282977289985865, + 6.007054439996864, + 6.072305907997361, + 6.140759629997774, + 6.216167343998677, + 6.270534751998639, + 6.333724023999821, + 6.4196597109985305, + 6.517789431996789, + 6.613042006996693, + 6.698048931997619, + 6.769445060999715, + 6.85497652399863, + 6.9473433069979365, + 7.052009528997587, + 7.136220876996958, + 7.2039461889980885, + 7.276498281997192, + 7.3455763530000695, + 7.40767997399962, + 7.4788782999967225, + 7.543097873996885, + 7.601105503999861, + 7.663331413998094, + 7.728805228998681, + 7.853427913996711 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.64534199889749, + "p95": 98.12972099825856, + "max": 154.2187439990812 + } + }, + { + "start": 29453.066593082, + "first": 29453.320190749, + "last": 29460.92016921, + "end": 29460.920514166, + "elapsed_s": 7.853921083999012, + "chunk_offsets_s": [ + 0.2535976670005766, + 0.4911066860004212, + 0.575705317001848, + 0.6699134959999355, + 0.7407863010012079, + 0.807181557000149, + 0.8671886660013115, + 0.9326105389991426, + 0.997363487000257, + 1.0832848169993667, + 1.159815310002159, + 1.249258137999277, + 1.3342578090014285, + 1.415045210000244, + 1.4819094679987757, + 1.549545810001291, + 1.631094052001572, + 1.7135221770004136, + 1.7862610859992856, + 1.869561330000579, + 1.9466716520000773, + 2.013366153001698, + 2.105338222001592, + 2.2040373970012297, + 2.292519745999016, + 2.370593567000469, + 2.438421180999285, + 2.4998560230014846, + 2.5558931970008416, + 2.609909888000402, + 2.670428637000441, + 2.7450171490017965, + 2.8177866600017296, + 2.876141650998761, + 2.935543214000063, + 3.003223126001103, + 3.0631130280016805, + 3.130899618001422, + 3.2002483249998477, + 3.2683978310014936, + 3.340748991999135, + 3.398394161999022, + 3.4586091860001034, + 3.5327071270003216, + 3.590477465000731, + 3.650161254001432, + 3.714822093999828, + 3.7806716710001638, + 3.8513260969994008, + 3.9308962460017938, + 4.022417457999836, + 4.08161061699866, + 4.1417694029987615, + 4.204452926001977, + 4.258857070999511, + 4.312658425998961, + 4.373189868001646, + 4.43743451499904, + 4.498021280000103, + 4.574996912000643, + 4.635545835000812, + 4.7020539910008665, + 4.800861431998783, + 4.894227509001212, + 4.997071672001766, + 5.09143312499873, + 5.183666919001553, + 5.250828340998851, + 5.339133427998604, + 5.42730075900181, + 5.503096082000411, + 5.558428166001249, + 5.625846482002089, + 5.704858482000418, + 5.789743383000314, + 5.861162091001461, + 5.928478666999581, + 6.007684211999731, + 6.072475139000744, + 6.141070694000518, + 6.21630549800102, + 6.270651058999647, + 6.3338949819990376, + 6.419932020999113, + 6.518379540000751, + 6.61332484200102, + 6.698220753001806, + 6.769495504999213, + 6.855090559001837, + 6.947328048001509, + 7.052175544999045, + 7.136497346000397, + 7.204269648002082, + 7.276660235002055, + 7.346023379999679, + 7.408517667001433, + 7.479365967999911, + 7.5433841470003244, + 7.601385156001925, + 7.663374533000024, + 7.729376448998664, + 7.853576127999986 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.65442599923699, + "p95": 98.69917499963776, + "max": 237.50901899984456 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.99134728692454, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.814588254001137, + "end_to_end_tps": 163.79621784226813, + "timing_valid": true, + "steady_tps": 171.29271598074453, + "per_stream_tps": [ + 34.5, + 34.1, + 33.7, + 33.4, + 33.0 + ], + "ttft_s": [ + 0.42, + 0.34, + 0.25, + 0.17, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.4202514130010968, + "decode_tps": 34.49391818453174, + "prefill_tps": null, + "itl_mean_ms": 28.99061784313138, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.34960099981981, + "round_gap_p50_ms": 69.01607100007823, + "round_gap_p95_ms": 98.46476099846768, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33748894600284984, + "decode_tps": 34.10728078632423, + "prefill_tps": null, + "itl_mean_ms": 29.3192531607786, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.15737899648957, + "round_gap_p50_ms": 68.84901999728754, + "round_gap_p95_ms": 98.08809000242036, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25405887799934135, + "decode_tps": 33.73406863364597, + "prefill_tps": null, + "itl_mean_ms": 29.643622619615225, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.18591199896764, + "round_gap_p50_ms": 68.77679100216483, + "round_gap_p95_ms": 98.7666610017186, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17073861000244506, + "decode_tps": 33.365854652413866, + "prefill_tps": null, + "itl_mean_ms": 29.970759341171394, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.9400710008631, + "round_gap_p50_ms": 68.92767799945432, + "round_gap_p95_ms": 98.69786800118163, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08642297999904258, + "decode_tps": 33.00309323470372, + "prefill_tps": null, + "itl_mean_ms": 30.300190133343943, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.18300100110355, + "round_gap_p50_ms": 69.02099700164399, + "round_gap_p95_ms": 98.59244000108447, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.64488861960811, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.9400710008631, + "round_gap_p50_ms": 68.92767799945432, + "round_gap_p95_ms": 98.69786800118163, + "round_gap_mean_ms": 74.84600592079275, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 29460.92325372, + "first": 29461.343505133, + "last": 29468.736112683, + "end": 29468.736218915, + "elapsed_s": 7.812965195000288, + "chunk_offsets_s": [ + 0.4202514130010968, + 0.48972788700120873, + 0.5744087690000015, + 0.6683701289985038, + 0.7391856879985426, + 0.7958622379992448, + 0.8525695770003949, + 0.9240862500009825, + 0.9931023210010608, + 1.0796843780008203, + 1.1552060619978874, + 1.2465556629977073, + 1.3309502880001673, + 1.409296933998121, + 1.4753534390001732, + 1.5415494250009942, + 1.6237869280012092, + 1.7062568369983637, + 1.773368311998638, + 1.8551179700007197, + 1.937874567000108, + 2.006291694000538, + 2.09745773800023, + 2.195922498998698, + 2.284043774998281, + 2.3618812220011023, + 2.428738037000585, + 2.4916124289993604, + 2.547406850000698, + 2.602292260999093, + 2.662388758999441, + 2.73743758099954, + 2.801871277999453, + 2.857708850999188, + 2.9227812530007213, + 2.9953112619987223, + 3.056364518997725, + 3.1223680549992423, + 3.1914076100001694, + 3.258276284999738, + 3.3303064049978275, + 3.3869931299996097, + 3.4465514889998303, + 3.519014450997929, + 3.5765795099978277, + 3.6358084919993416, + 3.701585202001297, + 3.763033348001045, + 3.830288835000829, + 3.9090685109986225, + 4.008082607000688, + 4.067437401998177, + 4.128741161999642, + 4.191732914998283, + 4.2455918960004055, + 4.301547496997955, + 4.360416285999236, + 4.426783033999527, + 4.486906461999752, + 4.564129275000596, + 4.623119185998803, + 4.689352801000496, + 4.782134419998329, + 4.870327721000649, + 4.980512982998334, + 5.079033654998057, + 5.17112864199953, + 5.237684109000838, + 5.324871726999845, + 5.415487280999514, + 5.491565070999059, + 5.545819374998246, + 5.613798538000992, + 5.693569729999581, + 5.772202110998478, + 5.839031696999882, + 5.907382763998612, + 5.99064754400024, + 6.053781831000379, + 6.119936978000624, + 6.196309081999061, + 6.250659959998302, + 6.310322053999698, + 6.39605990400014, + 6.493499199001235, + 6.589114120000886, + 6.674614907999057, + 6.741852937000658, + 6.821128027000668, + 6.916620371997851, + 7.029294997999386, + 7.114634419998765, + 7.181458389997715, + 7.254385756001284, + 7.324178799000947, + 7.387529103998531, + 7.459482921000017, + 7.523310788001254, + 7.582484304999525, + 7.642545082999277, + 7.709154717998899, + 7.812858962999599 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.01607100007823, + "p95": 98.46476099846768, + "max": 112.67462600153522 + } + }, + { + "start": 29460.922974323, + "first": 29461.260463269, + "last": 29468.736872825, + "end": 29468.73697098, + "elapsed_s": 7.813996657001553, + "chunk_offsets_s": [ + 0.33748894600284984, + 0.48954223100008676, + 0.5747424010005489, + 0.6685804640001152, + 0.73951548200057, + 0.7960095260023081, + 0.8526366709993454, + 0.9245767180000257, + 0.9936208910003188, + 1.0800437610014342, + 1.1554464020009618, + 1.2467586570019193, + 1.331280066002364, + 1.4094187920018157, + 1.475680817002285, + 1.5418786750014988, + 1.6238165810027567, + 1.7064822550019016, + 1.7732384420014569, + 1.8553127119994315, + 1.938324154001748, + 2.0068499760018312, + 2.0977853240001423, + 2.1958734140025626, + 2.2843864749993372, + 2.362085016000492, + 2.4292516320019786, + 2.491683953001484, + 2.54772445600247, + 2.602754100000311, + 2.662919354999758, + 2.7377396570009296, + 2.8023439290009264, + 2.8582240930008993, + 2.9236714239996218, + 2.9959619080000266, + 3.056696824001847, + 3.1226046520023374, + 3.191453671999625, + 3.258499015999405, + 3.3304018410017306, + 3.3874572080021608, + 3.4467821520011057, + 3.5193404860001465, + 3.5767731640007696, + 3.6358987130006426, + 3.702001219000522, + 3.7633648850023746, + 3.830622100002074, + 3.9099136120021285, + 4.009017161002703, + 4.067672864002816, + 4.128863227000693, + 4.191604597002879, + 4.245911997000803, + 4.301643091999722, + 4.360734739999316, + 4.427096482002526, + 4.487394068000867, + 4.564360946002125, + 4.623711951000587, + 4.689662570002838, + 4.781819948999328, + 4.870669206000457, + 4.980272691002028, + 5.078227499001514, + 5.171632033001515, + 5.23775497799943, + 5.324970009001845, + 5.415886808001233, + 5.491783787001623, + 5.546061553999607, + 5.613997325999662, + 5.693886041000951, + 5.772543084000063, + 5.838899332000437, + 5.9077392040017, + 5.990476825001679, + 6.053731833999336, + 6.120316049000394, + 6.196832178000477, + 6.250817162999738, + 6.310653415002889, + 6.3962791960002505, + 6.493631900000764, + 6.589517677002732, + 6.675006806002784, + 6.74223987699952, + 6.821539933000167, + 6.916838944001938, + 7.029836743000487, + 7.115167944000859, + 7.181836869000108, + 7.255055851001089, + 7.32469258600031, + 7.388206413001171, + 7.459806508999463, + 7.523835083000449, + 7.582706907000102, + 7.642919642999914, + 7.709520161999535, + 7.813898502001393 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.84901999728754, + "p95": 98.08809000242036, + "max": 152.05328499723691 + } + }, + { + "start": 29460.922882919, + "first": 29461.176941797, + "last": 29468.736065565, + "end": 29468.737360127, + "elapsed_s": 7.814477208001335, + "chunk_offsets_s": [ + 0.25405887799934135, + 0.4899466669994581, + 0.5745855009990919, + 0.6688027140007762, + 0.7398545979995106, + 0.7961973890014633, + 0.85249083799863, + 0.9247270429987111, + 0.994000935999793, + 1.080195205999189, + 1.1556339130002016, + 1.2467209589995036, + 1.3310259390018473, + 1.4095298200008983, + 1.4755742810011725, + 1.5416600220014516, + 1.6236102910006593, + 1.706284201998642, + 1.773669827998674, + 1.855440933999489, + 1.9381623910012422, + 2.0068973010020272, + 2.0974324029994023, + 2.196199064001121, + 2.284107749001123, + 2.36210862200096, + 2.429240915000264, + 2.492127302000881, + 2.547735490999912, + 2.602713809999841, + 2.6629575959996146, + 2.7377266049988975, + 2.802333052000904, + 2.8579572189992177, + 2.923407620000944, + 2.99601487900145, + 3.056903415999841, + 3.1226556559995515, + 3.1914324470017164, + 3.2582737090015144, + 3.3310174440011906, + 3.3876289170002565, + 3.44700115500018, + 3.5191886229986267, + 3.5769079429992416, + 3.636425822001911, + 3.7020196430021315, + 3.76333299199905, + 3.830500561001827, + 3.9093812069986598, + 4.008591227000579, + 4.067581057999632, + 4.128860539000016, + 4.191608929999347, + 4.245725347998814, + 4.301649361001182, + 4.360502796000219, + 4.426696745998925, + 4.486962039998616, + 4.564266085999407, + 4.623423900000489, + 4.689640082000551, + 4.781825993999519, + 4.87053644000116, + 4.980818545001966, + 5.078879441000026, + 5.171966784000688, + 5.2377797680019285, + 5.3251601760021, + 5.4159387389991025, + 5.491988876001415, + 5.546244779001427, + 5.614446032999695, + 5.693604484000389, + 5.7728426799985755, + 5.839145683999959, + 5.907314501000656, + 5.991283171999385, + 6.05400817500049, + 6.120363101999828, + 6.196858549999888, + 6.251089026001864, + 6.310824069001683, + 6.396502822000912, + 6.493908593001834, + 6.589533686001232, + 6.675385924001603, + 6.742275094999059, + 6.821823583999503, + 6.916555883999536, + 7.0298749829999, + 7.114621511998848, + 7.181890095998824, + 7.25481122400015, + 7.324360840000736, + 7.3879776670000865, + 7.459688633000042, + 7.5237320810010715, + 7.5825149480006075, + 7.642967576000956, + 7.709414458000538, + 7.8131826460012235 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.77679100216483, + "p95": 98.7666610017186, + "max": 235.88778900011675 + } + }, + { + "start": 29460.923182515, + "first": 29461.093921125, + "last": 29468.736464757, + "end": 29468.736527245, + "elapsed_s": 7.813344730002427, + "chunk_offsets_s": [ + 0.17073861000244506, + 0.48923054300030344, + 0.5747024970005441, + 0.6682194670029276, + 0.7394369840003492, + 0.7961322310002288, + 0.8527909960030229, + 0.9245240820018807, + 0.9939008310029749, + 1.0801572530035628, + 1.155441476003034, + 1.2470168940017174, + 1.3311456700030249, + 1.409761860002618, + 1.475896782001655, + 1.5419580050001969, + 1.624190261001786, + 1.7065124350010592, + 1.7736345620032807, + 1.8556464550019882, + 1.9380220460006967, + 2.006520039001771, + 2.0977849880000576, + 2.19549392800036, + 2.284587518002809, + 2.3621923030004837, + 2.428676877003454, + 2.491763763002382, + 2.5476098930012085, + 2.602252036002028, + 2.6625891940020665, + 2.737301617002231, + 2.802698321000207, + 2.858166132002225, + 2.9231755020009587, + 2.9954586130006646, + 3.0565591170015978, + 3.1226126810033747, + 3.191540359002829, + 3.2579268840017903, + 3.3298812890025147, + 3.3871765320000122, + 3.446840610002255, + 3.519344516000274, + 3.5768245210019813, + 3.636239559000387, + 3.7018561540025985, + 3.762709825001366, + 3.831017836000683, + 3.909520675002568, + 4.008650134001073, + 4.067838017002941, + 4.129095810003491, + 4.19168650100255, + 4.2460374210022565, + 4.301259943000332, + 4.361029091000091, + 4.426938799002528, + 4.4871267460002855, + 4.56366227000035, + 4.623422526001377, + 4.689610414003255, + 4.782007813002565, + 4.870099472002039, + 4.980346263000683, + 5.079044131001865, + 5.170984202002728, + 5.237904025001626, + 5.32463020400246, + 5.415763159002381, + 5.4917473690002225, + 5.546058932002779, + 5.614408464000007, + 5.694085069000721, + 5.7726468370019575, + 5.838906050001242, + 5.907300060000125, + 5.990489637002611, + 6.053929550002067, + 6.12040347200309, + 6.196509709003294, + 6.250902923002286, + 6.310932637003134, + 6.396620266001264, + 6.494043551003415, + 6.589532712001528, + 6.6750341080005455, + 6.7422672910033725, + 6.821480836002593, + 6.91638372700254, + 7.029431986000418, + 7.1146488310005225, + 7.181758084003377, + 7.254294735001167, + 7.324542868002027, + 7.387881481001386, + 7.459703509000974, + 7.523811285002012, + 7.582660046002275, + 7.643011913001828, + 7.7094471030031855, + 7.813282242001151 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.92767799945432, + "p95": 98.69786800118163, + "max": 318.4919329978584 + } + }, + { + "start": 29460.922771873, + "first": 29461.009194853, + "last": 29468.735743337, + "end": 29468.735819052, + "elapsed_s": 7.813047178999113, + "chunk_offsets_s": [ + 0.08642297999904258, + 0.4895419429994945, + 0.5747515340008249, + 0.6689875400006713, + 0.7395555299990519, + 0.796278509998956, + 0.8527298830013024, + 0.9246624929983227, + 0.9936834899999667, + 1.0800518850010121, + 1.155577727000491, + 1.2469001229983405, + 1.3312017759999435, + 1.4095773070002906, + 1.4756289010001638, + 1.5417163530000835, + 1.6236541469988879, + 1.7063513930006593, + 1.7739111119990412, + 1.8554767290006566, + 1.938701080998726, + 2.006677450001007, + 2.097420679998322, + 2.1960131199994066, + 2.2841750199986564, + 2.3624102670000866, + 2.429420623000624, + 2.4920363780001935, + 2.547806264999963, + 2.602897259999736, + 2.662803350998729, + 2.7378786920016864, + 2.8023047919996316, + 2.8581148230005056, + 2.9233768490012153, + 2.9957328119999147, + 3.0566913059992658, + 3.1224824269993405, + 3.192034744999546, + 3.2589322249987163, + 3.3309681030004867, + 3.387702681000519, + 3.447077895001712, + 3.5191983000004257, + 3.5769338380014233, + 3.636222444998566, + 3.701952228999289, + 3.7633876600011718, + 3.830658629001846, + 3.9096455219987547, + 4.008819268001389, + 4.067632254998898, + 4.1289122000016505, + 4.192037166001683, + 4.245891317001224, + 4.301497291999112, + 4.360683399001573, + 4.427029786998901, + 4.487164283000311, + 4.564321216999815, + 4.62347592900187, + 4.68969441399895, + 4.781877415000054, + 4.870939661999728, + 4.980527792999055, + 5.078583203001472, + 5.171559708000132, + 5.238050751999253, + 5.324897381000483, + 5.415898643001128, + 5.491924404999736, + 5.546203931000491, + 5.614754026999435, + 5.693780081001023, + 5.772584012000152, + 5.838891301998956, + 5.907220888999291, + 5.990537170000607, + 6.053780055000971, + 6.120570910999959, + 6.1967293199995765, + 6.251081686001271, + 6.310900600001332, + 6.396659850001015, + 6.4936508200007665, + 6.589678911001101, + 6.675340181998763, + 6.742046223000216, + 6.821838043000753, + 6.9168688360005035, + 7.030022191000171, + 7.114338980001776, + 7.181735627000307, + 7.25499696100087, + 7.324704661001306, + 7.388134838998667, + 7.459730842001591, + 7.524002122998354, + 7.5828752419984085, + 7.642977748000703, + 7.7094105719988875, + 7.812971464001748 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.02099700164399, + "p95": 98.59244000108447, + "max": 403.1189630004519 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/receipts/flashnext-cuda-sampled-heads/B2.json b/docs/receipts/flashnext-cuda-sampled-heads/B2.json new file mode 100644 index 0000000000..71e9e7e698 --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/B2.json @@ -0,0 +1,23728 @@ +{ + "label": "B2", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "code", + "temperature": 1.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.0, + "per_stream_tps": 67.0, + "ttft_s_max": 0.1, + "request": { + "ttft_s": 0.097, + "decode_tps": 67.005, + "prefill_tps": 144.146, + "itl_mean_ms": 14.924, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.381, + "round_gap_p50_ms": 67.031, + "round_gap_p95_ms": 82.238, + "tokens_per_chunk": 4.414 + }, + "pooled": { + "aggregate_prefill_tps": 144.146, + "itl_mean_ms": 14.924, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.381, + "round_gap_p50_ms": 67.031, + "round_gap_p95_ms": 82.238, + "tokens_per_chunk": 4.414 + }, + "timing_valid": true, + "end_to_end_tps": 65.639832796987, + "makespan_s": 3.9000708729981852, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.17572557956719, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.896772158997919, + "end_to_end_tps": 65.69539853873111, + "timing_valid": true, + "per_stream_tps": [ + 67.2 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.10067078999782098, + "decode_tps": 67.17572557956719, + "prefill_tps": 139.06715145776676, + "itl_mean_ms": 14.886329717652794, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.38137799990363, + "round_gap_p50_ms": 65.58843300081207, + "round_gap_p95_ms": 82.90688399938517, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.886329717652794, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.38137799990363, + "round_gap_p50_ms": 65.58843300081207, + "round_gap_p95_ms": 82.90688399938517, + "round_gap_mean_ms": 66.59673821055198, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 139.06715145776676 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 30220.106314361, + "first": 30220.206985151, + "last": 30224.002999229, + "end": 30224.00308652, + "elapsed_s": 3.896772158997919, + "chunk_offsets_s": [ + 0.10067078999782098, + 0.16113166399736656, + 0.2160554490001232, + 0.27547741799935466, + 0.3420454029983375, + 0.4037148569987039, + 0.4650296049985627, + 0.5335795040009543, + 0.6093508119993203, + 0.6841127160005271, + 0.7431285109996679, + 0.8005086230004963, + 0.855471419999958, + 0.9120757809978386, + 0.9849905929986562, + 1.0628291569992143, + 1.1391038369984017, + 1.223124428997835, + 1.3030520069987688, + 1.3726582969975425, + 1.4494234929989034, + 1.5181139009982871, + 1.5977245249996486, + 1.6765726310004538, + 1.7576694050003425, + 1.826257385997451, + 1.8803732319975097, + 1.9344726179988356, + 1.9825902779994067, + 2.027915963997657, + 2.0813372880002134, + 2.1413081799983047, + 2.2047333359987533, + 2.283114713998657, + 2.348703146999469, + 2.3929608580001513, + 2.4410010270003113, + 2.493439202000445, + 2.5437693919993762, + 2.6024456869999995, + 2.6843613529999857, + 2.7655900809986633, + 2.830886637999356, + 2.880520544000319, + 2.944248077998054, + 3.0212638119992334, + 3.1041706959986186, + 3.1878761389998544, + 3.267683333997411, + 3.3326254620005784, + 3.391497500000696, + 3.452859819997684, + 3.512523590998171, + 3.5905332990005263, + 3.6707233369997994, + 3.746003689997451, + 3.8255876019975403, + 3.8966848679992836 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 65.58843300081207, + "p95": 82.90688399938517, + "max": 84.02059199943324 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.0048957486881, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.902924588001042, + "end_to_end_tps": 65.59183869117884, + "timing_valid": true, + "per_stream_tps": [ + 67.0 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09712369699991541, + "decode_tps": 67.0048957486881, + "prefill_tps": 144.1460779650119, + "itl_mean_ms": 14.924282603925688, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.25924699864117, + "round_gap_p50_ms": 67.03056699916488, + "round_gap_p95_ms": 82.2379639976134, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.924282603925688, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.25924699864117, + "round_gap_p50_ms": 67.03056699916488, + "round_gap_p95_ms": 82.2379639976134, + "round_gap_mean_ms": 66.76652743861492, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 144.1460779650119 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 30224.003754767, + "first": 30224.100878464, + "last": 30227.906570528, + "end": 30227.906679355, + "elapsed_s": 3.902924588001042, + "chunk_offsets_s": [ + 0.09712369699991541, + 0.15328595800019684, + 0.20929172899923287, + 0.2735493629988923, + 0.34331181300149183, + 0.40530968999883044, + 0.4667265940006473, + 0.5358036809993791, + 0.6107595020002918, + 0.6859084419993451, + 0.7459755560012127, + 0.8023909869989438, + 0.8592669140016369, + 0.9173403380009404, + 0.9904076710008667, + 1.0680917659992701, + 1.1398309350006457, + 1.222138200999325, + 1.3042722249992948, + 1.376585633999639, + 1.4535856779984897, + 1.522362357001839, + 1.6015602760016918, + 1.679819523000333, + 1.7586980939995556, + 1.826867386000231, + 1.8817742179999186, + 1.937410247999651, + 1.9852715070010163, + 2.0288994789989374, + 2.0826334330013196, + 2.1383260570000857, + 2.198962049998954, + 2.2811618380001164, + 2.3476143830012006, + 2.393083138998918, + 2.4409280639993085, + 2.4934829880003235, + 2.5446761439998227, + 2.604050416000973, + 2.685996455998975, + 2.7652958469989244, + 2.829446782998275, + 2.8827738649997627, + 2.945563821998803, + 3.023537747001683, + 3.1057757109992963, + 3.1846575160016073, + 3.267419959000108, + 3.334450525999273, + 3.3947580810017826, + 3.4566007139983412, + 3.516139914998348, + 3.594913662000181, + 3.674307798999507, + 3.752221548998932, + 3.8303404250000312, + 3.9028157610009657 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 67.03056699916488, + "p95": 82.2379639976134, + "max": 82.76244299850077 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 66.99682032378489, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.9000708729981852, + "end_to_end_tps": 65.639832796987, + "timing_valid": true, + "per_stream_tps": [ + 67.0 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09381450299770222, + "decode_tps": 66.99682032378489, + "prefill_tps": 149.230657868996, + "itl_mean_ms": 14.926081494124054, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.84341500175651, + "round_gap_p50_ms": 68.97585499973502, + "round_gap_p95_ms": 81.21636800206034, + "tokens_per_chunk": 4.413793103448276 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.926081494124054, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.84341500175651, + "round_gap_p50_ms": 68.97585499973502, + "round_gap_p95_ms": 81.21636800206034, + "round_gap_mean_ms": 66.77457510529182, + "tokens_per_chunk": 4.413793103448276, + "aggregate_prefill_tps": 149.230657868996 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + } + ], + "client_intervals": [ + { + "start": 30227.907340655, + "first": 30228.001155158, + "last": 30231.807305939, + "end": 30231.807411528, + "elapsed_s": 3.9000708729981852, + "chunk_offsets_s": [ + 0.09381450299770222, + 0.14775086000008741, + 0.2013988889993925, + 0.26277991099777864, + 0.3331078749979497, + 0.39712521199908224, + 0.46191291499781073, + 0.5310372539970558, + 0.6059325649985112, + 0.680959923996852, + 0.7402842589981447, + 0.7964382649988693, + 0.8520526779975626, + 0.9086406239985081, + 0.9833132109997678, + 1.0602711100000306, + 1.1325332669985073, + 1.213922229999298, + 1.2928302959990106, + 1.3687978249981825, + 1.4489513819971762, + 1.5180690749984933, + 1.5969124900002498, + 1.6744939499985776, + 1.754376228000183, + 1.8230635309992067, + 1.8775674699973024, + 1.931914771997981, + 1.9820895140001085, + 2.0254049629984365, + 2.0793629820000206, + 2.134348677998787, + 2.1941241819986317, + 2.273339441999269, + 2.3438948759976483, + 2.3915267719967233, + 2.441283886997553, + 2.494096078997245, + 2.544557065997651, + 2.603025312997488, + 2.6842416809995484, + 2.7642350389978674, + 2.826619862997177, + 2.877705540999159, + 2.941152518000308, + 3.020462069998757, + 3.1030031469999813, + 3.183173669000098, + 3.2601497629984806, + 3.3291256179982156, + 3.391303966000123, + 3.455969260998245, + 3.515761629998451, + 3.5954981180002505, + 3.6743818830000237, + 3.7515042909981275, + 3.827547392997076, + 3.899965283999336 + ], + "chunk_gaps_ms": { + "count": 57, + "p50": 68.97585499973502, + "p95": 81.21636800206034, + "max": 82.54107700122404 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.3, + "per_stream_tps": 47.3, + "ttft_s_max": 0.21, + "request": { + "ttft_s": 0.146, + "decode_tps": 47.274, + "prefill_tps": 106.008, + "itl_mean_ms": 21.441, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.528, + "round_gap_p50_ms": 80.958, + "round_gap_p95_ms": 101.12, + "tokens_per_chunk": 3.73 + }, + "pooled": { + "aggregate_prefill_tps": 144.923, + "itl_mean_ms": 21.441, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.027, + "round_gap_p50_ms": 80.068, + "round_gap_p95_ms": 100.159, + "tokens_per_chunk": 3.631 + }, + "timing_valid": true, + "end_to_end_tps": 81.3404056727676, + "makespan_s": 6.2945346259984944, + "steady_tps": 92.0, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.29920031127185, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.2945346259984944, + "end_to_end_tps": 81.3404056727676, + "timing_valid": true, + "steady_tps": 92.45733237681878, + "per_stream_tps": [ + 52.8, + 41.8 + ], + "ttft_s": [ + 0.1, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09734276700328337, + "decode_tps": 52.75328186837636, + "prefill_tps": 143.82167705924962, + "itl_mean_ms": 18.956166603910628, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.10226399934618, + "round_gap_p50_ms": 84.45456700064824, + "round_gap_p95_ms": 102.02507100257208, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.1932053829987126, + "decode_tps": 41.794893207800186, + "prefill_tps": 72.46174916406594, + "itl_mean_ms": 23.926368109809403, + "itl_p50_ms": 0.0, + "itl_p95_ms": 90.80639699823223, + "round_gap_p50_ms": 77.32048799880431, + "round_gap_p95_ms": 99.70754800087889, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 21.441267356860017, + "itl_p50_ms": 0.0, + "itl_p95_ms": 90.80639699823223, + "round_gap_p50_ms": 80.42541299801087, + "round_gap_p95_ms": 100.1586970014614, + "round_gap_mean_ms": 78.66939821581732, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 144.92349832813187 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 30231.808567463, + "first": 30231.90591023, + "last": 30236.739732714, + "end": 30236.739853352, + "elapsed_s": 4.931285889000719, + "chunk_offsets_s": [ + 0.09734276700328337, + 0.2519130100008624, + 0.30692899400310125, + 0.3686286350020964, + 0.45087359700119123, + 0.5241703760002565, + 0.5940921030014579, + 0.6785466700021061, + 0.7631113190000178, + 0.8530402860014874, + 0.9371977460032213, + 0.9998337830002129, + 1.0799215160004678, + 1.1511162720016728, + 1.2289241130019946, + 1.315048481003032, + 1.40255074200104, + 1.495466465003119, + 1.5846116190004977, + 1.6649550500005716, + 1.7515844510016905, + 1.8314164220028033, + 1.9225186860021495, + 2.0087594330034335, + 2.098178516000189, + 2.1872750820002693, + 2.252015685000515, + 2.3243367220020446, + 2.3942276580019097, + 2.4664314670008025, + 2.5433836820011493, + 2.6161061660022824, + 2.692253042001539, + 2.7846712999999, + 2.867699887003255, + 2.9272161799999594, + 2.9857693370031484, + 3.0565104310007882, + 3.141810054999951, + 3.229943029000424, + 3.3060543870014953, + 3.3885952300006466, + 3.4906707360023574, + 3.5809834340034286, + 3.675514626000222, + 3.7756733230016835, + 3.8756380470003933, + 3.9619812260025355, + 4.049641378001979, + 4.1344013110028754, + 4.196077497002989, + 4.268868305000069, + 4.349661937001656, + 4.439711443003034, + 4.5381133440023405, + 4.6347117080003954, + 4.7367367790029675, + 4.836583741001959, + 4.931165251000493 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 84.50960799927998, + "p95": 102.02507100257208, + "max": 154.57024299757904 + } + }, + { + "start": 30231.808382019, + "first": 30232.001587402, + "last": 30238.10281127, + "end": 30238.102916645, + "elapsed_s": 6.2945346259984944, + "chunk_offsets_s": [ + 0.1932053829987126, + 0.251960178000445, + 0.3073248010005045, + 0.36912192800082266, + 0.45096647899845266, + 0.5241105319983035, + 0.5945601729981718, + 0.6791377039990039, + 0.7634263910003938, + 0.8534853720011597, + 0.9376891530009743, + 1.0002760280003713, + 1.080186273000436, + 1.1517420620002667, + 1.2293382579991885, + 1.3153508649993455, + 1.402920913998969, + 1.4959092210010567, + 1.5850578760000644, + 1.6654832889980753, + 1.7519816200001515, + 1.8319253839981684, + 1.9230905129988969, + 2.0094222340012493, + 2.0982685329981905, + 2.1878274949995102, + 2.252573884001322, + 2.325076534998516, + 2.394696536997799, + 2.4668936320013017, + 2.5438548940001056, + 2.6164319529998465, + 2.6927183080006216, + 2.785271249998914, + 2.868093991000933, + 2.927565607998986, + 2.9862056429992663, + 3.056972760001372, + 3.142318547998002, + 3.2306742510008917, + 3.3067820129981556, + 3.3887366369999654, + 3.4906111050004256, + 3.581417501998658, + 3.675860618997831, + 3.776329321997764, + 3.876036869998643, + 3.9625178999995114, + 4.050027000001137, + 4.134376377998706, + 4.196735612000339, + 4.269249051998486, + 4.350014806001127, + 4.44027310599995, + 4.538903669999854, + 4.634561581999151, + 4.737088592999498, + 4.836883781001234, + 4.931203011001344, + 4.98811030700017, + 5.037123709000298, + 5.082072860997869, + 5.137215351998748, + 5.19832821700038, + 5.2573339470000064, + 5.317249753999931, + 5.395919540998875, + 5.4614433879978606, + 5.509661805997894, + 5.555514496998512, + 5.601955582998926, + 5.662241285001073, + 5.739561772999878, + 5.817682585999137, + 5.890717718000815, + 5.963824768001359, + 6.040458695999405, + 6.1045266699984495, + 6.149629522999021, + 6.196818632000941, + 6.2429291820008075, + 6.2944292510001105 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 77.32048799880431, + "p95": 99.70754800087889, + "max": 102.52701100034756 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.03813462065042, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.318029977999686, + "end_to_end_tps": 81.03791874727719, + "timing_valid": true, + "steady_tps": 91.95035529548339, + "per_stream_tps": [ + 53.9, + 41.0 + ], + "ttft_s": [ + 0.21, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.2126662219998252, + "decode_tps": 53.94520364513966, + "prefill_tps": 65.83085864952972, + "itl_mean_ms": 18.537329223523983, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.00801600309205, + "round_gap_p50_ms": 83.68762000100105, + "round_gap_p95_ms": 100.54022399708629, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.10125567999784835, + "decode_tps": 41.01906731032521, + "prefill_tps": 138.26384851000452, + "itl_mean_ms": 24.378906337255568, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.25571599765681, + "round_gap_p50_ms": 78.32064899776014, + "round_gap_p95_ms": 99.28779400070198, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 21.458117780389774, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.13658899898292, + "round_gap_p50_ms": 80.0682160006545, + "round_gap_p95_ms": 99.6992020009202, + "round_gap_mean_ms": 78.73122351078263, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 131.6003287111402 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 30238.104052112, + "first": 30238.316718334, + "last": 30243.043737286, + "end": 30243.043923798, + "elapsed_s": 4.939871685997787, + "chunk_offsets_s": [ + 0.2126662219998252, + 0.2721166369992716, + 0.3304500169979292, + 0.39023505200020736, + 0.4681765440000163, + 0.539233843999682, + 0.6070370029992773, + 0.6907246230002784, + 0.7748684539983515, + 0.864151800997206, + 0.9488146019975829, + 1.0113402799979667, + 1.0895889780003927, + 1.1652402879990404, + 1.245308503999695, + 1.3322225429983519, + 1.4194844429985096, + 1.50474665299771, + 1.5915783189993817, + 1.6734505629974592, + 1.7606939929974033, + 1.840376465999725, + 1.931616704998305, + 2.0190747719971114, + 2.1110827880002034, + 2.204755683997064, + 2.2742499989981297, + 2.3456879849982215, + 2.4129869829994277, + 2.479702647000522, + 2.5527683459986292, + 2.623937340998964, + 2.699752439999429, + 2.793826506996993, + 2.8778255309989618, + 2.9363804469976458, + 2.9949344839988044, + 3.063591299000109, + 3.150751217999641, + 3.2436589199969603, + 3.3203228389975266, + 3.4033532769972226, + 3.499423264998768, + 3.5876607299978787, + 3.6820437889982713, + 3.7816253139972105, + 3.8831515169986233, + 3.970577523999964, + 4.058674771000369, + 4.147278393997112, + 4.21298806899722, + 4.287488138998015, + 4.367763733996981, + 4.451122938000481, + 4.547024523999426, + 4.643161969997891, + 4.74563145600041, + 4.8461716799974965, + 4.939685173998441 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 83.84332200148492, + "p95": 100.54022399708629, + "max": 102.46948600251926 + } + }, + { + "start": 30238.103952908, + "first": 30238.205208588, + "last": 30244.421829704, + "end": 30244.421982886, + "elapsed_s": 6.318029977999686, + "chunk_offsets_s": [ + 0.10125567999784835, + 0.27243479199751164, + 0.3302833859997918, + 0.3906134229982854, + 0.4685338140006934, + 0.5390633280003385, + 0.6069961760003935, + 0.6903600549994735, + 0.7744253499986371, + 0.863939847000438, + 0.9487466940008744, + 1.0111535510004614, + 1.0894741999982216, + 1.1651343499979703, + 1.2448552169989853, + 1.331944041998213, + 1.4193715519977559, + 1.5045359139985521, + 1.591206205997878, + 1.6730369680008152, + 1.7602869459988142, + 1.8402096520003397, + 1.9315827159989567, + 2.018762343999697, + 2.11089893299868, + 2.205136283999309, + 2.273973938998097, + 2.3454941229974793, + 2.413475486999232, + 2.479466885997681, + 2.5526993639978173, + 2.62374976099818, + 2.699966742999095, + 2.794151719997899, + 2.877803033999953, + 2.93665399299789, + 2.995260928000789, + 3.063323993999802, + 3.1507383919997665, + 3.2439941079974233, + 3.3208265989997017, + 3.403169193999929, + 3.4989408209985413, + 3.587385319999157, + 3.681879511997977, + 3.781167305998679, + 3.883701832997758, + 3.970926830999815, + 4.058887236998999, + 4.147616542999458, + 4.212825324997539, + 4.28688671999771, + 4.367434717998549, + 4.451140036999277, + 4.546446748998278, + 4.642573690998688, + 4.746142657997552, + 4.8458418599984725, + 4.94044817699978, + 4.9965429699987, + 5.047241730997484, + 5.091356246000942, + 5.151032593999844, + 5.215281695000158, + 5.274617549999675, + 5.334780679997493, + 5.414168631999928, + 5.474723041999823, + 5.520521698999801, + 5.565722210998501, + 5.611897696999222, + 5.672696649999125, + 5.75200864100043, + 5.8298615729981975, + 5.903150188998552, + 5.975445051997667, + 6.053786722997756, + 6.118485000999499, + 6.168099038997752, + 6.216446564998478, + 6.26465299299889, + 6.317876795998018 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 78.32064899776014, + "p95": 99.28779400070198, + "max": 171.1791119996633 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 82.398077908408, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.284916904998681, + "end_to_end_tps": 81.46487976520152, + "timing_valid": true, + "steady_tps": 91.91984528235737, + "per_stream_tps": [ + 53.9, + 41.2 + ], + "ttft_s": [ + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18982410799799254, + "decode_tps": 53.92730410722042, + "prefill_tps": 73.75248669756981, + "itl_mean_ms": 18.543482129419264, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.02700199946412, + "round_gap_p50_ms": 83.59596800073632, + "round_gap_p95_ms": 102.4076089997834, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09510908000083873, + "decode_tps": 41.199038954204, + "prefill_tps": 147.19940514487723, + "itl_mean_ms": 24.272410847048626, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.95469999758643, + "round_gap_p50_ms": 77.89618199967663, + "round_gap_p95_ms": 101.69896200022777, + "tokens_per_chunk": 3.1219512195121952 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 21.407946488233947, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.02700199946412, + "round_gap_p50_ms": 79.0590050019091, + "round_gap_p95_ms": 102.4076089997834, + "round_gap_mean_ms": 78.54714179136195, + "tokens_per_chunk": 3.631205673758865, + "aggregate_prefill_tps": 147.18792682314518 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + } + ], + "client_intervals": [ + { + "start": 30244.423767681, + "first": 30244.613591789, + "last": 30249.342179732, + "end": 30249.342384257, + "elapsed_s": 4.918616576000204, + "chunk_offsets_s": [ + 0.18982410799799254, + 0.2483806509990245, + 0.30459613199855085, + 0.36398303199894144, + 0.44255509799768333, + 0.5130973669984087, + 0.5816436370005249, + 0.6648333859993727, + 0.7497044049996475, + 0.8434715539988247, + 0.93119510500037, + 0.993529948998912, + 1.0714246450006613, + 1.1402968029979093, + 1.2162937049979519, + 1.3034845330003009, + 1.3899423250004475, + 1.4785176279983716, + 1.5653482390007412, + 1.645410967998032, + 1.7343066629982786, + 1.8170106399993529, + 1.9125784409989137, + 2.0000596530007897, + 2.0905906410007447, + 2.1794855049993203, + 2.2458842050000385, + 2.3170504359986808, + 2.3833639129989024, + 2.4515138370006753, + 2.5275804939992668, + 2.5985237790009705, + 2.6760510239982978, + 2.7682827789976727, + 2.8569478489989706, + 2.917088875998161, + 2.974128137000662, + 3.044672306998109, + 3.128690370998811, + 3.216679158998886, + 3.2931063229989377, + 3.373756920998858, + 3.471019141998113, + 3.558854371000052, + 3.652720549998776, + 3.7544067230010114, + 3.8610262629990757, + 3.95205326499854, + 4.038584820998949, + 4.122778444998403, + 4.184733394999057, + 4.2575434379978105, + 4.336370681998233, + 4.419966649998969, + 4.514941369998269, + 4.611784766999335, + 4.715308050999738, + 4.8177156599995214, + 4.918412050999905 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 83.80701600071916, + "p95": 102.4076089997834, + "max": 106.61953999806428 + } + }, + { + "start": 30244.423358795, + "first": 30244.518467875, + "last": 30250.707932641, + "end": 30250.7082757, + "elapsed_s": 6.284916904998681, + "chunk_offsets_s": [ + 0.09510908000083873, + 0.24904298600085895, + 0.30525325599955977, + 0.36472188200059463, + 0.44322768399797496, + 0.5136809869982244, + 0.5817423019980197, + 0.6654110589988704, + 0.7497950289980508, + 0.8441500070002803, + 0.9315146900007676, + 0.9939968670005328, + 1.0718930490002094, + 1.1412469009992492, + 1.2176750760008872, + 1.3040689359986573, + 1.3905838669998047, + 1.479154357999505, + 1.565873477000423, + 1.646091565999086, + 1.7344731240009423, + 1.8173442119987158, + 1.9128041180010769, + 2.000398244999815, + 2.090924165000615, + 2.1801571030009654, + 2.246246250000695, + 2.3174081799988926, + 2.383822310999676, + 2.451859078999405, + 2.527823881999211, + 2.5988885890001256, + 2.6766821130004246, + 2.768636812998011, + 2.857301036001445, + 2.9174525669986906, + 2.9751409640011843, + 3.045032481000817, + 3.129537332999462, + 3.2170332099994994, + 3.2937587070009613, + 3.37473004200001, + 3.471383790998516, + 3.5593456870010414, + 3.653319986999122, + 3.75501894899935, + 3.8616395290009677, + 3.952231136998307, + 4.038749006998842, + 4.12296884999887, + 4.185198971001228, + 4.25807286700001, + 4.336867449997953, + 4.420167077998485, + 4.51540120700156, + 4.612336083999253, + 4.715554890000931, + 4.8183153069985565, + 4.918993103998218, + 4.975271357001475, + 5.024425323001196, + 5.070827567000379, + 5.12723173500126, + 5.188337632000184, + 5.246100258998922, + 5.305198628000653, + 5.383388646998355, + 5.445444412998768, + 5.490141714999481, + 5.535124984999129, + 5.581459804001497, + 5.642228403001354, + 5.720382644998608, + 5.799441650000517, + 5.8772936329987715, + 5.954502312000841, + 6.0294851350008685, + 6.093443362999096, + 6.137482253001508, + 6.184278038999764, + 6.231476165001368, + 6.2845738459982385 + ], + "chunk_gaps_ms": { + "count": 81, + "p50": 77.89618199967663, + "p95": 101.69896200022777, + "max": 153.93390600002022 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 118.4, + "per_stream_tps": 32.3, + "ttft_s_max": 0.39, + "request": { + "ttft_s": 0.237, + "decode_tps": 32.34, + "prefill_tps": 61.171, + "itl_mean_ms": 30.922, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.516, + "round_gap_p50_ms": 110.192, + "round_gap_p95_ms": 141.834, + "tokens_per_chunk": 3.633 + }, + "pooled": { + "aggregate_prefill_tps": 146.577, + "itl_mean_ms": 29.843, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.417, + "round_gap_p50_ms": 108.388, + "round_gap_p95_ms": 141.569, + "tokens_per_chunk": 3.618 + }, + "timing_valid": true, + "end_to_end_tps": 117.54741167866443, + "makespan_s": 8.711378543997853, + "steady_tps": 133.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 112.48366572995765, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 9.162899311002548, + "end_to_end_tps": 111.75502046283648, + "timing_valid": true, + "steady_tps": 126.61724263740358, + "per_stream_tps": [ + 36.0, + 28.4, + 32.4, + 30.4 + ], + "ttft_s": [ + 0.28, + 0.19, + 0.38, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.28242064700316405, + "decode_tps": 36.01887819750497, + "prefill_tps": 49.57144652332431, + "itl_mean_ms": 27.76321890194987, + "itl_p50_ms": 0.0, + "itl_p95_ms": 134.88293399859685, + "round_gap_p50_ms": 123.08149199816398, + "round_gap_p95_ms": 151.61645300031523, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.18870883799900184, + "decode_tps": 28.417321969305934, + "prefill_tps": 74.18836419348865, + "itl_mean_ms": 35.18980434117325, + "itl_p50_ms": 0.0, + "itl_p95_ms": 134.87621899912483, + "round_gap_p50_ms": 112.11492100119358, + "round_gap_p95_ms": 146.27300800202647, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3758774669986451, + "decode_tps": 32.43478048776826, + "prefill_tps": 37.246180548647956, + "itl_mean_ms": 30.831101211772285, + "itl_p50_ms": 0.0, + "itl_p95_ms": 134.32336799814948, + "round_gap_p50_ms": 115.46100900159217, + "round_gap_p95_ms": 145.9979379978904, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09481266399961896, + "decode_tps": 30.44165202153146, + "prefill_tps": 147.65959956632233, + "itl_mean_ms": 32.84972836864101, + "itl_p50_ms": 0.0, + "itl_p95_ms": 134.7309890006727, + "round_gap_p50_ms": 115.73608699836768, + "round_gap_p95_ms": 149.933910000982, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 31.658463205884104, + "itl_p50_ms": 0.0, + "itl_p95_ms": 134.7309890006727, + "round_gap_p50_ms": 115.61107600209652, + "round_gap_p95_ms": 149.5125020010164, + "round_gap_mean_ms": 115.74061817204941, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 148.59694117994735 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 30250.709628896, + "first": 30250.992049543, + "last": 30258.071670363, + "end": 30258.071866158, + "elapsed_s": 7.362237262001145, + "chunk_offsets_s": [ + 0.28242064700316405, + 0.44444106300215935, + 0.5132411570011755, + 0.6077988290016947, + 0.7127055119999568, + 0.8163545610004803, + 0.9237024260000908, + 1.052547648003383, + 1.1799860280007124, + 1.3112748200001079, + 1.4460486570023932, + 1.5761087090031651, + 1.706695781002054, + 1.8146343780026655, + 1.9349191590008559, + 2.070406834001915, + 2.2200893700028246, + 2.37170582300314, + 2.516756491000706, + 2.663256623000052, + 2.795402000003378, + 2.933186182002828, + 3.086825097001565, + 3.217349483002181, + 3.352232417000778, + 3.493539471000986, + 3.6255145800023456, + 3.7485960720005096, + 3.8540555810031947, + 3.9436782990014763, + 4.044079576000513, + 4.150559796002199, + 4.26329383400298, + 4.389250087002438, + 4.511733629002265, + 4.625693768000929, + 4.731021272000362, + 4.837027202000172, + 4.951209672002733, + 5.07384335900133, + 5.186036206003337, + 5.3098293540024315, + 5.440255517001788, + 5.568536467002559, + 5.68419227800041, + 5.810490824002045, + 5.9356275480022305, + 6.0510523920020205, + 6.166439742002694, + 6.271240732003207, + 6.3551559440020355, + 6.462586341000133, + 6.575969049001287, + 6.68199433600239, + 6.804149583000253, + 6.9371724120028375, + 7.082830373001343, + 7.221553473002132, + 7.362041467000381 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 123.43731999862939, + "p95": 151.61645300031523, + "max": 162.0204159989953 + } + }, + { + "start": 30250.710221999, + "first": 30250.898930837, + "last": 30259.872330944, + "end": 30259.872434312, + "elapsed_s": 9.162212313000055, + "chunk_offsets_s": [ + 0.18870883799900184, + 0.44326129500041134, + 0.5128383869996469, + 0.6073117239975545, + 0.7113144510003622, + 0.8159457309993741, + 0.9230352049999055, + 1.0521873569996387, + 1.1792608659998223, + 1.3103757589997258, + 1.4453020229993854, + 1.575197832997219, + 1.7059755919981399, + 1.8142356549979013, + 1.9346828710004047, + 2.0695468630001415, + 2.2188006809992658, + 2.371713896998699, + 2.5159411239983456, + 2.662214132000372, + 2.79466903499997, + 2.9322609629998624, + 3.0857123019995925, + 3.2164555489980557, + 3.3513317679971806, + 3.4930500669979665, + 3.6252046739973594, + 3.7474279749985726, + 3.852937765997922, + 3.9431424169997626, + 4.043059032999736, + 4.149577157997555, + 4.262198606997117, + 4.388850456998625, + 4.511285171000054, + 4.625397341998905, + 4.7305038569975295, + 4.836228394997306, + 4.9511206280003535, + 5.073482388997945, + 5.185597309999139, + 5.308769665, + 5.439931704997434, + 5.568536209997546, + 5.684147285999643, + 5.80943410799955, + 5.934702192000259, + 6.050172410999949, + 6.165601280998089, + 6.27035729599811, + 6.354927232998307, + 6.46160571099972, + 6.575802260998898, + 6.681269423999765, + 6.8036219049972715, + 6.936519288999989, + 7.08229247899726, + 7.220687988999998, + 7.360885247999249, + 7.449530214998958, + 7.528403758999048, + 7.608736879999924, + 7.6960644889986725, + 7.795792403998348, + 7.896259132998239, + 7.976560724997398, + 8.069488971999817, + 8.158637620999798, + 8.237887973999023, + 8.307376329998078, + 8.380088908998005, + 8.470994729999802, + 8.539382427999954, + 8.619757054999354, + 8.689717635999841, + 8.75495093399877, + 8.82830424200074, + 8.90616372599834, + 8.969657042998733, + 9.014914461000444, + 9.061430999998265, + 9.108343691998016, + 9.162108944998181 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 112.36818500037771, + "p95": 146.27300800202647, + "max": 254.5524570014095 + } + }, + { + "start": 30250.710515897, + "first": 30251.086393364, + "last": 30258.948324173, + "end": 30258.948522752, + "elapsed_s": 8.238006855001004, + "chunk_offsets_s": [ + 0.3758774669986451, + 0.44301711799926125, + 0.5124412730001495, + 0.6069702149980003, + 0.7114408509987697, + 0.8150924740002665, + 0.9228932049991272, + 1.0512275100008992, + 1.1788736960006645, + 1.3103266660000372, + 1.4446841969984234, + 1.5751245499996003, + 1.7050596900007804, + 1.813246097000956, + 1.9341705000006186, + 2.0690836379981192, + 2.2185961399991356, + 2.371370771998045, + 2.5159601579980517, + 2.661847408999165, + 2.794324278998829, + 2.9327876919996925, + 3.085639134998928, + 3.216209088001051, + 3.3505324559992005, + 3.492536588997609, + 3.6250651240006846, + 3.74748213699786, + 3.8525648709983216, + 3.9424186649994226, + 4.043006186999264, + 4.149495214998751, + 4.261946965998504, + 4.388680962998478, + 4.511123495998618, + 4.624986345999787, + 4.7304588469996816, + 4.836002046999056, + 4.950893049001024, + 5.073259717999463, + 5.185345108999172, + 5.308566101000906, + 5.439820105999388, + 5.568294034997962, + 5.683953553998435, + 5.809216697998636, + 5.934481264997885, + 6.049942273999477, + 6.165263750001031, + 6.270139118998486, + 6.354454696000175, + 6.461470513997483, + 6.575175483998464, + 6.681040567000309, + 6.803389497999888, + 6.935807997000666, + 7.081805934998556, + 7.22101782000027, + 7.360854984999605, + 7.449419865999516, + 7.528438138000638, + 7.608651664999343, + 7.6956064849982795, + 7.795429099998728, + 7.8962830229975225, + 7.976086716000282, + 8.069496762000199, + 8.158026843997504, + 8.237808276000578 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 115.56026400103292, + "p95": 145.9979379978904, + "max": 152.8514429992356 + } + }, + { + "start": 30250.709535001, + "first": 30250.804347665, + "last": 30259.181028399, + "end": 30259.181262505, + "elapsed_s": 8.471727504002047, + "chunk_offsets_s": [ + 0.09481266399961896, + 0.44427506400097627, + 0.5133925910013204, + 0.6080243340002198, + 0.7122827990024234, + 0.8163782860028732, + 0.9234975920007855, + 1.0527940490028413, + 1.1800308250021772, + 1.311133487000916, + 1.4458644760015886, + 1.5757790400020895, + 1.7064044190010463, + 1.8144634680029412, + 1.9350668690021848, + 2.069975002999854, + 2.219908913000836, + 2.3718998180011113, + 2.5164848870008427, + 2.663154554000357, + 2.7957598870016227, + 2.933433401001821, + 3.0871973270004673, + 3.217760341001849, + 3.351698573002068, + 3.4937016510011745, + 3.625774927000748, + 3.748885037002765, + 3.853914244002226, + 3.943914344003133, + 4.0442316200023924, + 4.150713103001181, + 4.263144805001502, + 4.389510259003146, + 4.512041526002577, + 4.626041619001626, + 4.731248284002504, + 4.837174241001776, + 4.951363139000023, + 5.0740324320031505, + 5.186180704000435, + 5.309978441000567, + 5.440248095001152, + 5.5689402320022054, + 5.684676319000573, + 5.810295347000647, + 5.935307954001473, + 6.051247524999781, + 6.16643028800172, + 6.271220214002824, + 6.355312913001399, + 6.462930151003093, + 6.576409205001255, + 6.681858269002987, + 6.8041104069998255, + 6.937065685000562, + 7.082644501002505, + 7.2217336900030205, + 7.361243410003226, + 7.450687996002671, + 7.529541042000346, + 7.609233839000808, + 7.6966817770007765, + 7.796694368000317, + 7.896698421001929, + 7.977172705002886, + 8.070877275000385, + 8.159510167002736, + 8.238360245002696, + 8.308145893999608, + 8.38065042800008, + 8.471493398003076 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 115.73608699836768, + "p95": 149.933910000982, + "max": 349.4624000013573 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 118.42730510292671, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 8.711378543997853, + "end_to_end_tps": 117.54741167866443, + "timing_valid": true, + "steady_tps": 135.7824376850439, + "per_stream_tps": [ + 38.0, + 30.3, + 34.5, + 32.2 + ], + "ttft_s": [ + 0.19, + 0.29, + 0.39, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.1923840300005395, + "decode_tps": 37.9962615448443, + "prefill_tps": 72.7711130698361, + "itl_mean_ms": 26.318378686276038, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.5308369982231, + "round_gap_p50_ms": 111.95759899783297, + "round_gap_p95_ms": 141.88405800086912, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2900880209999741, + "decode_tps": 30.281995155305985, + "prefill_tps": 48.26121379207606, + "itl_mean_ms": 33.02292318822926, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.54469800306833, + "round_gap_p50_ms": 106.57967299994198, + "round_gap_p95_ms": 139.83380799982115, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3900568210010533, + "decode_tps": 34.49918023049429, + "prefill_tps": 35.89220658690184, + "itl_mean_ms": 28.98619599998746, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.41666800098028, + "round_gap_p50_ms": 108.12009900109842, + "round_gap_p95_ms": 139.85879900064901, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09818112799985101, + "decode_tps": 32.21193946374691, + "prefill_tps": 142.59359497296919, + "itl_mean_ms": 31.044389647057887, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.40778400216368, + "round_gap_p50_ms": 108.31501899883733, + "round_gap_p95_ms": 141.78306999747292, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 29.84297188038766, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.41666800098028, + "round_gap_p50_ms": 108.38842500015744, + "round_gap_p95_ms": 141.56866899793386, + "round_gap_mean_ms": 109.10333805733123, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 143.42480028836744 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 30259.874569738, + "first": 30260.066953768, + "last": 30266.778140333, + "end": 30266.778344709, + "elapsed_s": 6.90377497100053, + "chunk_offsets_s": [ + 0.1923840300005395, + 0.45900576599888154, + 0.529269036000187, + 0.6061432889982825, + 0.6938476389987045, + 0.7761896480005817, + 0.8648894110010588, + 0.9713122030007071, + 1.0773021200002404, + 1.1854962309989787, + 1.2933348619990284, + 1.3954203790017345, + 1.5044743229991582, + 1.5919907850002346, + 1.689171781999903, + 1.7970558050001273, + 1.9202597859984962, + 2.0433391320002556, + 2.159616802000528, + 2.2736431779994746, + 2.3826270799982012, + 2.4975876429998607, + 2.626737457001582, + 2.7550633280006878, + 2.888131686999259, + 3.030015745000128, + 3.1548902559989074, + 3.276062976998219, + 3.387144388998422, + 3.4809333010016417, + 3.5812359689989535, + 3.685363190001226, + 3.797550086001138, + 3.923080922999361, + 4.046309151999594, + 4.154251496998768, + 4.2588290970015805, + 4.370275484001468, + 4.491524909000873, + 4.613852196998778, + 4.725708514000871, + 4.847813648000738, + 4.97908392699901, + 5.101186585001415, + 5.213144183999248, + 5.337546672999451, + 5.472638767998433, + 5.588610135000636, + 5.70466111299902, + 5.810119876998215, + 5.8933101699985855, + 6.000943340000958, + 6.108205459000601, + 6.210242353001377, + 6.335804144000576, + 6.47737281299851, + 6.623923333001585, + 6.763540469000873, + 6.903570595000929 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 112.07224749887246, + "p95": 141.88405800086912, + "max": 266.62173599834205 + } + }, + { + "start": 30259.87404115, + "first": 30260.164129171, + "last": 30268.584974584, + "end": 30268.585293415, + "elapsed_s": 8.71125226499862, + "chunk_offsets_s": [ + 0.2900880209999741, + 0.45932870699834893, + 0.5298529930005316, + 0.6063017369997397, + 0.6944318200003181, + 0.7761867699991853, + 0.8653203079993546, + 0.9718999809992965, + 1.0778752049991454, + 1.1851863150004647, + 1.2939585679996526, + 1.3960243219989934, + 1.5052325910000945, + 1.5920191619989055, + 1.689636782000889, + 1.7977914799994323, + 1.920716166998318, + 2.044015457999194, + 2.1599522910000815, + 2.274216391000664, + 2.3832492350011307, + 2.497956053000962, + 2.627352097999392, + 2.7557557389991416, + 2.888351159999729, + 3.030433689000347, + 3.1560846130014397, + 3.2769084530009422, + 3.3873210409983585, + 3.4811680099992373, + 3.5816662719989836, + 3.686162696998508, + 3.798242881999613, + 3.9236782810003206, + 4.047305567000876, + 4.1551198230008595, + 4.259207723000145, + 4.370882930001244, + 4.4920104869997886, + 4.614442182999483, + 4.726026603999344, + 4.8481781230002525, + 4.979674825000984, + 5.101808806000918, + 5.213733594999212, + 5.33810551300121, + 5.471698198998638, + 5.5893796179989295, + 5.705272541999875, + 5.810462077999546, + 5.8939758659980725, + 6.001377607000904, + 6.108922370000073, + 6.210253205998015, + 6.335797904001083, + 6.478485635998368, + 6.624575246998575, + 6.764409054998396, + 6.903801428998122, + 6.994766218998848, + 7.073006990998692, + 7.148940950999531, + 7.23302288999912, + 7.330266395998478, + 7.43668595499912, + 7.519947476001107, + 7.6135776070004795, + 7.700157865001529, + 7.782524905000173, + 7.851360559001478, + 7.924169138997968, + 8.014983138000389, + 8.079680273000122, + 8.156625561998226, + 8.22508775100141, + 8.294267966000916, + 8.371643374000996, + 8.451437182000518, + 8.518422353998176, + 8.562886227999115, + 8.610755663001328, + 8.65779763899991, + 8.710933433998434 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 106.94539150063065, + "p95": 139.83380799982115, + "max": 169.24068599837483 + } + }, + { + "start": 30259.874306563, + "first": 30260.264363384, + "last": 30267.655843364, + "end": 30267.656036804, + "elapsed_s": 7.781730241000332, + "chunk_offsets_s": [ + 0.3900568210010533, + 0.45944667200092226, + 0.5296781079996435, + 0.6058605440011888, + 0.6938383499982592, + 0.7763843590000761, + 0.8645120760011196, + 0.9720269350000308, + 1.077806231998693, + 1.1854581119987415, + 1.2930286089977017, + 1.3959682989989233, + 1.5051830439988407, + 1.5920187160008936, + 1.6892690719978418, + 1.7973891709989402, + 1.9203104349981004, + 2.043413765000878, + 2.1600340949989913, + 2.273810770999262, + 2.3825451810007507, + 2.497435248998954, + 2.626658244000282, + 2.7550221210003656, + 2.888841689000401, + 3.0298815599999216, + 3.155298228000902, + 3.2769645799999125, + 3.3871700300005614, + 3.481284911998955, + 3.581563963998633, + 3.6853467959990667, + 3.798188492000918, + 3.923627841999405, + 4.046895483999833, + 4.154304925999895, + 4.2592746279988205, + 4.370197193999047, + 4.4921086829999695, + 4.6142735369976435, + 4.725726990000112, + 4.847955385001114, + 4.97917926199807, + 5.101779553999222, + 5.213211890000821, + 5.337528964999365, + 5.472167764000915, + 5.588992319000681, + 5.705221305001032, + 5.810035460999643, + 5.893255334998685, + 6.000817733998701, + 6.108772415998828, + 6.210168360998068, + 6.335899222998705, + 6.47849888500059, + 6.624108415999217, + 6.763967214999866, + 6.90338400499968, + 6.99396066600093, + 7.072284072000912, + 7.148283636000997, + 7.232199334001052, + 7.330090651001228, + 7.436302939000598, + 7.519489597998472, + 7.613014710001153, + 7.699535391999234, + 7.781536800997856 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 108.42725450129365, + "p95": 139.85879900064901, + "max": 145.60953099862672 + } + }, + { + "start": 30259.873914871, + "first": 30259.972095999, + "last": 30267.888415359, + "end": 30267.888555827, + "elapsed_s": 8.014640956000221, + "chunk_offsets_s": [ + 0.09818112799985101, + 0.45974882699738373, + 0.5301428849998047, + 0.6059821700000612, + 0.6939059659998748, + 0.7759112669991737, + 0.8649518780002836, + 0.97255800199855, + 1.0783026829994924, + 1.185168326999701, + 1.2934833459985384, + 1.3959108639974147, + 1.5050297710004088, + 1.5923028830002295, + 1.6895607539991033, + 1.7976461250000284, + 1.920378973998595, + 2.0437232900003437, + 2.159950082997966, + 2.2746042010003293, + 2.382992626000487, + 2.4979449229977035, + 2.6267211390004377, + 2.755111924998346, + 2.8882974989974173, + 3.0303432359978615, + 3.155269232000137, + 3.2768486509994545, + 3.38736475299811, + 3.4810364170007233, + 3.5821911909988557, + 3.6855808359978255, + 3.798055150997243, + 3.9234629349994066, + 4.046804047997284, + 4.154773078000289, + 4.259730740999657, + 4.370683990000543, + 4.492013440998562, + 4.614601862998825, + 4.725890804998926, + 4.848026461997506, + 4.9794274189989665, + 5.10163379900041, + 5.21345528099846, + 5.338013474000036, + 5.472847230998013, + 5.589117900999554, + 5.704998225999589, + 5.809694060997572, + 5.892866590998892, + 6.000518925997312, + 6.108519304998481, + 6.21048424699984, + 6.336376330000348, + 6.478159399997821, + 6.624660481000319, + 6.763948971998616, + 6.90360931199757, + 6.994127674999618, + 7.072300859999814, + 7.148324599998887, + 7.232319524999184, + 7.330447741998796, + 7.436607392999576, + 7.519835607999994, + 7.613354324999818, + 7.699983718997828, + 7.781994114000554, + 7.8512589409983775, + 7.923958006998873, + 8.014500487999612 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 108.31501899883733, + "p95": 141.78306999747292, + "max": 361.5676989975327 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 118.62506134947809, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 8.694929008001054, + "end_to_end_tps": 117.76979421657353, + "timing_valid": true, + "steady_tps": 133.00170719764105, + "per_stream_tps": [ + 38.0, + 30.3, + 34.5, + 32.2 + ], + "ttft_s": [ + 0.19, + 0.29, + 0.38, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19074834600178292, + "decode_tps": 38.00858069466038, + "prefill_tps": 73.39513182394327, + "itl_mean_ms": 26.309848505879213, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.28402899988578, + "round_gap_p50_ms": 114.11910800234182, + "round_gap_p95_ms": 146.72745799907716, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2867061629985983, + "decode_tps": 30.328813495953995, + "prefill_tps": 48.83048154102096, + "itl_mean_ms": 32.97194597254537, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.45279899859452, + "round_gap_p50_ms": 105.7887919996574, + "round_gap_p95_ms": 138.83721299862373, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38136861699967994, + "decode_tps": 34.49119302566825, + "prefill_tps": 36.70989005372655, + "itl_mean_ms": 28.99290840000236, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.26043900288641, + "round_gap_p50_ms": 108.4261729993159, + "round_gap_p95_ms": 138.90101099968888, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09629452200169908, + "decode_tps": 32.24497648208493, + "prefill_tps": 145.38729419886394, + "itl_mean_ms": 31.012582705885755, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.50190500041936, + "round_gap_p50_ms": 108.29972599822213, + "round_gap_p95_ms": 139.31631399827893, + "tokens_per_chunk": 3.5555555555555554 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 29.821821396078175, + "itl_p50_ms": 0.0, + "itl_p95_ms": 125.27593299819273, + "round_gap_p50_ms": 108.35632400267059, + "round_gap_p95_ms": 139.31631399827893, + "round_gap_mean_ms": 109.02601370609226, + "tokens_per_chunk": 3.618374558303887, + "aggregate_prefill_tps": 146.57742945432867 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + } + ], + "client_intervals": [ + { + "start": 30268.586776619, + "first": 30268.777524965, + "last": 30275.486536334, + "end": 30275.487502437, + "elapsed_s": 6.900725818002684, + "chunk_offsets_s": [ + 0.19074834600178292, + 0.4483063910010969, + 0.51796436800214, + 0.594005670001934, + 0.6861783640015346, + 0.7707061680011975, + 0.859551392000867, + 0.9639206419997208, + 1.0691973499997403, + 1.1775536740024108, + 1.2872464210013277, + 1.386062370002037, + 1.4889219520009647, + 1.5779557740024757, + 1.6804155019999598, + 1.7945346100023016, + 1.9180137800030934, + 2.041999039000075, + 2.159024593001959, + 2.274658657002874, + 2.378106957003183, + 2.487629499002651, + 2.6152136710006744, + 2.7518576470029075, + 2.8874919490008324, + 3.025927402002708, + 3.150108503003139, + 3.2720033149998926, + 3.378238176002924, + 3.466042357002152, + 3.567670927001018, + 3.6779118520025804, + 3.7940011430000595, + 3.9192851719999453, + 4.040190253002947, + 4.148488572001952, + 4.254335388002801, + 4.3626008960018225, + 4.479066579002392, + 4.602344831000664, + 4.720022470002732, + 4.8454107840007055, + 4.977386337002827, + 5.098495797003125, + 5.210642505000578, + 5.337363950002327, + 5.46263988300052, + 5.580773984002008, + 5.700568708001811, + 5.809153154001251, + 5.8926155980007024, + 5.998818031002884, + 6.105861666001147, + 6.208731345999695, + 6.332094304001657, + 6.46708573000069, + 6.613813187999767, + 6.761257304002356, + 6.899759715000982 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 114.87658600162831, + "p95": 146.72745799907716, + "max": 257.558044999314 + } + }, + { + "start": 30268.586863762, + "first": 30268.873569925, + "last": 30277.281416148, + "end": 30277.281530318, + "elapsed_s": 8.694666555998992, + "chunk_offsets_s": [ + 0.2867061629985983, + 0.44805164499848615, + 0.5179514650008059, + 0.5940335400009644, + 0.6860212659994431, + 0.7706738859997131, + 0.85990121299983, + 0.9637870970000222, + 1.069939267999871, + 1.1775156189978588, + 1.2872929519980971, + 1.3861809019981592, + 1.4888791000012134, + 1.5776318559983338, + 1.6801112030007062, + 1.7943193089995475, + 1.9180843909998657, + 2.041653006999695, + 2.158871845000249, + 2.2744349610002246, + 2.3777980839986412, + 2.4876525740000943, + 2.6150037109982804, + 2.7515189049991022, + 2.887050392000674, + 3.0258876049992978, + 3.150092322997807, + 3.271968218999973, + 3.3777570109996304, + 3.465661687998363, + 3.567823822999344, + 3.677508732998831, + 3.7942158400001063, + 3.9193605020009272, + 4.040220746999694, + 4.148318026000197, + 4.253943348998291, + 4.362279356999352, + 4.478226438997808, + 4.601294935000624, + 4.71947683200051, + 4.8449564809998265, + 4.977005746997747, + 5.09814785799972, + 5.210386603001098, + 5.337010461000318, + 5.462463259998913, + 5.580777743998624, + 5.700582638000924, + 5.808823858998949, + 5.892467132998718, + 5.998522522000712, + 6.105852810000215, + 6.208363529000053, + 6.332226704998902, + 6.467122509999172, + 6.61388153200096, + 6.760888815999351, + 6.900558340999851, + 6.988543756000581, + 7.064953823999531, + 7.140048868001031, + 7.225457115000609, + 7.325355381999543, + 7.4258598659980635, + 7.505374526997912, + 7.599740528999973, + 7.690921665998758, + 7.775666616998933, + 7.843117079999502, + 7.916822848001175, + 8.004087650999281, + 8.068535071000952, + 8.145038727998326, + 8.213422299999365, + 8.281725711000036, + 8.356260659998952, + 8.431979282999237, + 8.496423958000378, + 8.541779574999964, + 8.587887227000465, + 8.63437325400082, + 8.694552385997667 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 105.92209050082602, + "p95": 138.83721299862373, + "max": 161.34548199988785 + } + }, + { + "start": 30268.587283328, + "first": 30268.968651945, + "last": 30276.361843587, + "end": 30276.36205261, + "elapsed_s": 7.774769281997578, + "chunk_offsets_s": [ + 0.38136861699967994, + 0.44743898899832857, + 0.5179210860005696, + 0.5940380989995901, + 0.6853887189972738, + 0.7705363120003312, + 0.8593782739990274, + 0.963321609000559, + 1.0688267769983213, + 1.1774178919986298, + 1.2870345459996315, + 1.3860866949980846, + 1.488573828999506, + 1.57736519899845, + 1.6799747339973692, + 1.7940997419973428, + 1.917349811999884, + 2.041408495999349, + 2.1589063219980744, + 2.2739126449996547, + 2.3782033339994086, + 2.4873354859992105, + 2.6150533959989843, + 2.750882724998519, + 2.8864039159998356, + 3.0253049269995245, + 3.150101186998654, + 3.271693744998629, + 3.3774934419998317, + 3.465190299000824, + 3.5675949199976458, + 3.677524506998452, + 3.7932491339997796, + 3.9190422469982877, + 4.039904683999339, + 4.148330856998655, + 4.25416320399745, + 4.361527776000003, + 4.47822911899857, + 4.601950755000871, + 4.7193814940001175, + 4.844974702999025, + 4.976130308998108, + 5.097872472997551, + 5.2102318159995775, + 5.33668761899753, + 5.4619480580004165, + 5.580058348998136, + 5.700114701998245, + 5.808335583998996, + 5.892241941997781, + 5.998487835997366, + 6.105199757999799, + 6.208451375998266, + 6.332070398999349, + 6.466333381998993, + 6.6130582810001215, + 6.760052484998596, + 6.8997550460007915, + 6.9885516529975575, + 7.0647924010008865, + 7.1398826650001865, + 7.225248250997538, + 7.324746444999619, + 7.425151901999925, + 7.505169547999685, + 7.5991422410006635, + 7.690662816999975, + 7.774560259000282 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 108.5086439998122, + "p95": 138.90101099968888, + "max": 146.9942039984744 + } + }, + { + "start": 30268.58660131, + "first": 30268.682895832, + "last": 30276.591104422, + "end": 30276.591213235, + "elapsed_s": 8.004611925000063, + "chunk_offsets_s": [ + 0.09629452200169908, + 0.448055161999946, + 0.51855660700312, + 0.5944396270024299, + 0.6861329840030521, + 0.7711761339996883, + 0.8595017910010938, + 0.963953627000592, + 1.0700994990002073, + 1.178156897000008, + 1.2870170470014273, + 1.3863555070020084, + 1.488875036000536, + 1.5779627760020958, + 1.680457264999859, + 1.7944462490013393, + 1.9178313350021199, + 2.0417049230018165, + 2.159060648002196, + 2.2742745310024475, + 2.3791126790019916, + 2.4875109960012196, + 2.615557080000144, + 2.7514671420030936, + 2.8870051370031433, + 3.0263214510014222, + 3.1504644800006645, + 3.2725220380016253, + 3.3778713650026475, + 3.4657084210011817, + 3.568202138001652, + 3.6773930250019475, + 3.794397352001397, + 3.919337423001707, + 4.040629762002936, + 4.1489294880011585, + 4.254441764001967, + 4.362057346999791, + 4.478688877003151, + 4.602425064000272, + 4.719582871002785, + 4.845738413001527, + 4.976625437000621, + 5.098253673000727, + 5.211147591002373, + 5.336991649000993, + 5.462493554001412, + 5.580565437001496, + 5.700426200000948, + 5.808834325001953, + 5.892659501001617, + 5.999114176000148, + 6.106174472002749, + 6.208492001002014, + 6.332643829002336, + 6.466748925002321, + 6.613343362001615, + 6.760946915001114, + 6.900042637000297, + 6.988923750999675, + 7.064949609000905, + 7.1406285150005715, + 7.225584114999947, + 7.325189091003267, + 7.425877893001598, + 7.505258556000626, + 7.599594417002663, + 7.690916699000809, + 7.7750782830007665, + 7.843163825000374, + 7.916868760999932, + 8.004503112002567 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 108.29972599822213, + "p95": 139.31631399827893, + "max": 351.7606399982469 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 1.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 132.7, + "per_stream_tps": 30.0, + "ttft_s_max": 0.48, + "request": { + "ttft_s": 0.285, + "decode_tps": 29.957, + "prefill_tps": 49.04, + "itl_mean_ms": 33.381, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.34, + "round_gap_p50_ms": 125.509, + "round_gap_p95_ms": 150.164, + "tokens_per_chunk": 3.71 + }, + "pooled": { + "aggregate_prefill_tps": 147.007, + "itl_mean_ms": 33.222, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.34, + "round_gap_p50_ms": 125.383, + "round_gap_p95_ms": 150.155, + "tokens_per_chunk": 3.657 + }, + "timing_valid": true, + "end_to_end_tps": 131.94666237101615, + "makespan_s": 9.700889564002864, + "steady_tps": 151.7, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 132.74766465778313, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 9.710812821002037, + "end_to_end_tps": 131.81182910164668, + "timing_valid": true, + "steady_tps": 152.19235925511202, + "per_stream_tps": [ + 33.5, + 26.8, + 30.4, + 29.2, + 31.3 + ], + "ttft_s": [ + 0.11, + 0.2, + 0.39, + 0.29, + 0.48 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.10597908400086453, + "decode_tps": 33.54992756514931, + "prefill_tps": 132.10153807222747, + "itl_mean_ms": 29.806323666664813, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.61329900100827, + "round_gap_p50_ms": 128.1195360024867, + "round_gap_p95_ms": 152.02178699837532, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2002656919976289, + "decode_tps": 26.813043750933552, + "prefill_tps": 69.90713117335024, + "itl_mean_ms": 37.29528095687319, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.59398499698727, + "round_gap_p50_ms": 120.12829900049837, + "round_gap_p95_ms": 149.5130250004877, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.388845401997969, + "decode_tps": 30.371851409653946, + "prefill_tps": 36.004026093828216, + "itl_mean_ms": 32.925223639219496, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.29046899991226, + "round_gap_p50_ms": 126.16493799941964, + "round_gap_p95_ms": 150.87022299849195, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2942839380011719, + "decode_tps": 29.21504257488169, + "prefill_tps": 47.57310268134392, + "itl_mean_ms": 34.22894207451107, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.5194930007856, + "round_gap_p50_ms": 125.29849700149498, + "round_gap_p95_ms": 150.76814100029878, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.48361958199893706, + "decode_tps": 31.34534170070802, + "prefill_tps": 28.94837289700724, + "itl_mean_ms": 31.902667054907628, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.34035900072195, + "round_gap_p50_ms": 126.28600199968787, + "round_gap_p95_ms": 149.92528799848515, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 33.23168747843524, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.34035900072195, + "round_gap_p50_ms": 126.16493799941964, + "round_gap_p95_ms": 150.51582799787866, + "round_gap_mean_ms": 122.81275807247806, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 144.7418644850362 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 30277.283185654, + "first": 30277.389164738, + "last": 30284.989777273, + "end": 30284.989927121, + "elapsed_s": 7.706741467001848, + "chunk_offsets_s": [ + 0.10597908400086453, + 0.552212339000107, + 0.625885928999196, + 0.7119345360006264, + 0.80613900300159, + 0.8941191849989991, + 0.9976761920006538, + 1.1162088550008775, + 1.2361686490003194, + 1.3546961630017904, + 1.4672155150001345, + 1.5753835030009213, + 1.6932792420011538, + 1.8012254550012585, + 1.903444800998841, + 2.0326314279991493, + 2.16454633900139, + 2.2972082529995532, + 2.424699426999723, + 2.551907928998844, + 2.6800274650013307, + 2.808085710999876, + 2.95891586200014, + 3.1077676040004008, + 3.246380903001409, + 3.396199598999374, + 3.5278882599995995, + 3.661283494999225, + 3.7875828959986393, + 3.8987009199991007, + 4.032449815000291, + 4.161882733998937, + 4.291266929001722, + 4.4276840679995075, + 4.565752080001403, + 4.690940817999945, + 4.8097544379998, + 4.927499812998576, + 5.0671356189996, + 5.205325740000262, + 5.334147613000823, + 5.463741760999255, + 5.60091200199895, + 5.732968390999304, + 5.861050347000855, + 5.9997757820019615, + 6.140641117999621, + 6.262714502001472, + 6.38387402599983, + 6.521492845000466, + 6.625199097001314, + 6.737108313001954, + 6.853454645999591, + 6.974317120999331, + 7.115860754998721, + 7.256895839000208, + 7.408917625998583, + 7.552806515999691, + 7.706591619000392 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 128.47070450152387, + "p95": 152.02178699837532, + "max": 446.2332549992425 + } + }, + { + "start": 30277.283291142, + "first": 30277.483556834, + "last": 30286.993853478, + "end": 30286.99392612, + "elapsed_s": 9.710634978000598, + "chunk_offsets_s": [ + 0.2002656919976289, + 0.5520047690006322, + 0.6259964849996322, + 0.7114985129992419, + 0.8059705619998567, + 0.8944632629973057, + 0.997529960997781, + 1.115873474998807, + 1.2360017739993054, + 1.3545174389983003, + 1.4672040980003658, + 1.57499242799895, + 1.6936142390004534, + 1.8000908939975488, + 1.904000703998463, + 2.0324756349982636, + 2.1647325459998683, + 2.2971480659980443, + 2.424835568999697, + 2.551757667999482, + 2.6797759529981704, + 2.808259621997422, + 2.95857390799938, + 3.1080869329998677, + 3.246336849999352, + 3.3956854610005394, + 3.5278968309976335, + 3.660920177000662, + 3.7873111560002144, + 3.898502143998485, + 4.032280701998388, + 4.162019884999609, + 4.2908164959990245, + 4.4271577920007985, + 4.565751776997786, + 4.690581704999204, + 4.809512822997931, + 4.927285989000666, + 5.067519087999244, + 5.205128883000725, + 5.333973909000633, + 5.464349465000851, + 5.6005069950006146, + 5.732917221997923, + 5.861157660998288, + 5.9999272620007105, + 6.140860061997955, + 6.26314455299871, + 6.383652782998979, + 6.521574510999926, + 6.624479617999896, + 6.736950934999186, + 6.853264503999526, + 6.974507139999332, + 7.115303963997576, + 7.256363098000293, + 7.409301322000829, + 7.5531665309972595, + 7.706428629000584, + 7.821670101999189, + 7.929300092997437, + 8.040894132998801, + 8.158150073999423, + 8.282524217000173, + 8.396034766999946, + 8.495266394998907, + 8.61838519199955, + 8.706748697997682, + 8.786625464999815, + 8.854937975000212, + 8.928539197000646, + 9.022721007997461, + 9.090169252998749, + 9.16657159999886, + 9.235209649999888, + 9.303009237999504, + 9.377525121999497, + 9.453317640000023, + 9.518101178997313, + 9.562780061998637, + 9.609768767997593, + 9.657989056999213, + 9.710562336000294 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 120.31826450038352, + "p95": 149.5130250004877, + "max": 351.73907700300333 + } + }, + { + "start": 30277.284062636, + "first": 30277.672908038, + "last": 30286.068840066, + "end": 30286.068938169, + "elapsed_s": 8.78487553299783, + "chunk_offsets_s": [ + 0.388845401997969, + 0.5518488519992388, + 0.6253852609988826, + 0.7109318379989418, + 0.8058126939977228, + 0.893147044997022, + 0.9970495220004523, + 1.1154928149990155, + 1.2355670189972443, + 1.35431730900018, + 1.4666758479979762, + 1.5747974169971712, + 1.6931460809973942, + 1.800154606000433, + 1.903061519999028, + 2.031794677997823, + 2.163585966998653, + 2.2958735159991193, + 2.4242982209980255, + 2.5509055349975824, + 2.679552736997721, + 2.8076602170003753, + 2.9585304399988672, + 3.1076196869980777, + 3.24591015599799, + 3.395510934999038, + 3.527294947998598, + 3.6607287999977416, + 3.7868937379971612, + 3.897877041999891, + 4.031699813000159, + 4.161959597000532, + 4.2903162949987745, + 4.42699889099822, + 4.565155378997588, + 4.690319187997375, + 4.80917761299861, + 4.926844794998033, + 5.066484245999163, + 5.204052769997361, + 5.333530907999375, + 5.463141718999395, + 5.600081551998301, + 5.732381204998092, + 5.860095986998203, + 5.9985241729991685, + 6.140152067000599, + 6.262517612998636, + 6.383292086000438, + 6.521190564999415, + 6.623541357999784, + 6.736561711997638, + 6.852637692998542, + 6.974064075999195, + 7.114674178999849, + 7.256551055997988, + 7.408296350000455, + 7.5521055110002635, + 7.705110638999031, + 7.821415168997191, + 7.928653933999158, + 8.039961149999726, + 8.157642393998685, + 8.281830821997573, + 8.39544241600015, + 8.49432031499964, + 8.617690564999066, + 8.706060290998721, + 8.78477742999894 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 126.38612599948829, + "p95": 150.87022299849195, + "max": 163.0034500012698 + } + }, + { + "start": 30277.283530802, + "first": 30277.57781474, + "last": 30286.306194969, + "end": 30286.306321182, + "elapsed_s": 9.022790380000515, + "chunk_offsets_s": [ + 0.2942839380011719, + 0.5517069039997295, + 0.6254979269979231, + 0.7118053519989189, + 0.8056708100011747, + 0.8941521429987915, + 0.9971690790007415, + 1.116098332000547, + 1.2358710400003474, + 1.3544464769984188, + 1.4670166299983975, + 1.5749435799989442, + 1.69261537499915, + 1.8000699180010997, + 1.9039741470005538, + 2.0320373270005803, + 2.163982539001154, + 2.2971555840013025, + 2.4251678660002653, + 2.551822704997903, + 2.680340768998576, + 2.807949013000325, + 2.958717154000624, + 3.107174748998659, + 3.246142634001444, + 3.3957632259989623, + 3.527768208998168, + 3.661047433000931, + 3.786867650000204, + 3.8980379620006715, + 4.031955029000528, + 4.162088680001034, + 4.291305998998723, + 4.427419607000047, + 4.565251855998213, + 4.690550352999708, + 4.8096248250003555, + 4.927295700999821, + 5.0673272230014845, + 5.204944821998652, + 5.333876292999776, + 5.46374841999932, + 5.60045367299972, + 5.7329628500010585, + 5.861095463998936, + 5.9996149569997215, + 6.140235861999827, + 6.262981759999093, + 6.38374817300064, + 6.521486202000233, + 6.624658720000298, + 6.73738889300148, + 6.853396368998801, + 6.974025701998471, + 7.11524544599888, + 7.256603652000194, + 7.409115678001399, + 7.5530817409999145, + 7.7055740489995515, + 7.821266489998379, + 7.929257597999822, + 8.040548838998802, + 8.158008548998623, + 8.282027027998993, + 8.395728041999973, + 8.49514885499957, + 8.618600690999301, + 8.706257824000204, + 8.785940177000157, + 8.854952214998775, + 8.928223710001475, + 9.022664167001494 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 125.29849700149498, + "p95": 150.76814100029878, + "max": 257.42296599855763 + } + }, + { + "start": 30277.283113299, + "first": 30277.766732881, + "last": 30285.90191298, + "end": 30285.902165009, + "elapsed_s": 8.619051710000349, + "chunk_offsets_s": [ + 0.48361958199893706, + 0.5522322300021187, + 0.6257472120014427, + 0.7119576410004811, + 0.8062660290015629, + 0.8942399270017631, + 0.9977900900012173, + 1.1158777759992518, + 1.2363328840001486, + 1.3548175760006416, + 1.467327414000465, + 1.5754068160022143, + 1.6930884440007503, + 1.8007219829996757, + 1.9043332700020983, + 2.0325228049987345, + 2.165119686000253, + 2.297045342002093, + 2.424556792000658, + 2.5521035860001575, + 2.6806313560009585, + 2.808312213001045, + 2.958828040998924, + 3.107958095999493, + 3.2462984550002147, + 3.3962237429987, + 3.5281429050010047, + 3.661268621999625, + 3.7875546239993128, + 3.8983707160004997, + 4.032573116001004, + 4.1622662150002725, + 4.290881781998905, + 4.427271464999649, + 4.566026139000314, + 4.691077391002182, + 4.809379139001976, + 4.927667583000584, + 5.068020850001631, + 5.204879476001224, + 5.334713561998797, + 5.464425497000775, + 5.601228404000722, + 5.73326256600194, + 5.8608693160022085, + 5.999724241999502, + 6.140795695999259, + 6.2630078780020995, + 6.3838812520007195, + 6.521386200998677, + 6.624743585998658, + 6.737447423001868, + 6.853502199999639, + 6.974249815000803, + 7.115339170999505, + 7.256683833002171, + 7.409046283999487, + 7.552715686000738, + 7.707309436002106, + 7.8220543040006305, + 7.929346045999409, + 8.040762237000308, + 8.158248539999477, + 8.282837789000041, + 8.39608295799917, + 8.49538796699926, + 8.618799681000382 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 126.89872599912633, + "p95": 149.92528799848515, + "max": 154.59375000136788 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 132.72504767351765, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 9.700889564002864, + "end_to_end_tps": 131.94666237101615, + "timing_valid": true, + "steady_tps": 151.6553310132638, + "per_stream_tps": [ + 34.0, + 27.1, + 30.4, + 29.9, + 30.0 + ], + "ttft_s": [ + 0.19, + 0.28, + 0.38, + 0.48, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18704321200129925, + "decode_tps": 33.974434948907096, + "prefill_tps": 74.84901403373436, + "itl_mean_ms": 29.43389644313035, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.70137899700785, + "round_gap_p50_ms": 127.80279700018582, + "round_gap_p95_ms": 157.13188000154332, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28329410799779, + "decode_tps": 27.080235306625514, + "prefill_tps": 49.41860633440783, + "itl_mean_ms": 36.92730098823542, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.3249540012912, + "round_gap_p50_ms": 121.40188799821772, + "round_gap_p95_ms": 146.58065199910197, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3827103120020183, + "decode_tps": 30.383166161945994, + "prefill_tps": 36.5811935580303, + "itl_mean_ms": 32.9129622196014, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.0456039987621, + "round_gap_p50_ms": 125.66635900293477, + "round_gap_p95_ms": 151.24910200029262, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.4751958340020792, + "decode_tps": 29.89203505104694, + "prefill_tps": 29.46153774559131, + "itl_mean_ms": 33.45372766666069, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.64101400010986, + "round_gap_p50_ms": 124.65745999725186, + "round_gap_p95_ms": 146.38089700019918, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.0940738590034016, + "decode_tps": 29.95693642811064, + "prefill_tps": 148.81923786600248, + "itl_mean_ms": 33.38125052939765, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.19907500146655, + "round_gap_p50_ms": 125.50887300312752, + "round_gap_p95_ms": 150.16426800139016, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 33.2218275694051, + "itl_p50_ms": 0.0, + "itl_p95_ms": 137.88497299901792, + "round_gap_p50_ms": 125.38267399941105, + "round_gap_p95_ms": 150.15458000198123, + "round_gap_mean_ms": 122.77631927823624, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 147.25586022228532 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 30286.99541072, + "first": 30287.182453932, + "last": 30294.688097525, + "end": 30294.688239698, + "elapsed_s": 7.692828978000762, + "chunk_offsets_s": [ + 0.18704321200129925, + 0.5441467500022554, + 0.6168993130013405, + 0.6992384889999812, + 0.7944406770002388, + 0.8818279110018921, + 0.9804773770010797, + 1.0941535099991597, + 1.2168135489991982, + 1.3434089690017572, + 1.4587425099998654, + 1.5652763889993366, + 1.6842127420022734, + 1.792613623001671, + 1.8977779170018039, + 2.0222735870011093, + 2.15188279600261, + 2.289584174999618, + 2.421185623999918, + 2.5489884210001037, + 2.675084914000763, + 2.8030084910024016, + 2.949513494000712, + 3.0953843659990525, + 3.237504110998998, + 3.3946359910005413, + 3.5275394800009963, + 3.6603006229997845, + 3.7859001889992214, + 3.8997228420012107, + 4.027052396999352, + 4.154984733999299, + 4.288661501999741, + 4.428119645999686, + 4.567028658999334, + 4.690484319999086, + 4.810056813999836, + 4.927897782999935, + 5.0594914960020105, + 5.198992696001369, + 5.335457839999435, + 5.467811794002046, + 5.605063430000882, + 5.738584588001686, + 5.8682726610022655, + 6.00398314999984, + 6.141485554002429, + 6.266206355001486, + 6.391577810001763, + 6.516503115999512, + 6.617003423001734, + 6.728882781000721, + 6.844073192001815, + 6.961427258000185, + 7.097537609999563, + 7.239221192001423, + 7.3995834269990155, + 7.542367063000711, + 7.692686804999539 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 127.86318700091215, + "p95": 157.13188000154332, + "max": 357.1035380009562 + } + }, + { + "start": 30286.995957298, + "first": 30287.279251406, + "last": 30296.695713158, + "end": 30296.696203438, + "elapsed_s": 9.700246140000672, + "chunk_offsets_s": [ + 0.28329410799779, + 0.5440700139988621, + 0.6164418599983037, + 0.6991199479998613, + 0.7946399209977244, + 0.8809611059987219, + 0.9798619600005622, + 1.0942293369989784, + 1.2160191789989767, + 1.34290780900119, + 1.4585495069986791, + 1.5653709199978039, + 1.6835403820005013, + 1.7919713169976603, + 1.897161541000969, + 2.02203014800034, + 2.150626058999478, + 2.2889510130007693, + 2.4205329260003055, + 2.5483123829981196, + 2.6738516459990933, + 2.80268894099936, + 2.949269592998462, + 3.0952205899993714, + 3.236833475999447, + 3.3944157980004093, + 3.527216620001127, + 3.660003341999982, + 3.785914537998906, + 3.8994860569982848, + 4.026842852999835, + 4.15402647999872, + 4.288191708998056, + 4.427887465000822, + 4.5665438219984935, + 4.690354981998098, + 4.809166373001062, + 4.927398429997993, + 5.059255239000777, + 5.198556890998589, + 5.335197938999045, + 5.466910402999929, + 5.604795375998947, + 5.738222865998978, + 5.867321693000122, + 6.0033137280006486, + 6.140760049998789, + 6.265496997999435, + 6.390766726999573, + 6.516132234999532, + 6.616503430999728, + 6.72810798899809, + 6.843374176998623, + 6.960720009999932, + 7.096538054000121, + 7.238917409998976, + 7.398978879999049, + 7.541596889997891, + 7.691751469999872, + 7.808897523998894, + 7.919887936001032, + 8.025290700999903, + 8.13981068800058, + 8.265184658997896, + 8.383570845999202, + 8.48392484299984, + 8.605326730998058, + 8.693383105997782, + 8.775866797997878, + 8.843722757999785, + 8.918238993999694, + 9.005913901000895, + 9.071208346998901, + 9.148797563000699, + 9.218232856001123, + 9.290417660999083, + 9.366899140000896, + 9.443035342999792, + 9.507551802998933, + 9.553141758999118, + 9.601070018998143, + 9.647324882000248, + 9.699755859997822 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 121.59586499910802, + "p95": 146.58065199910197, + "max": 260.77590600107214 + } + }, + { + "start": 30286.995702138, + "first": 30287.37841245, + "last": 30295.771217816, + "end": 30295.771419174, + "elapsed_s": 8.775717036001879, + "chunk_offsets_s": [ + 0.3827103120020183, + 0.5438008660021296, + 0.6161620320017391, + 0.6988011140019807, + 0.7945480599992152, + 0.8812894049988245, + 0.9794815749992267, + 1.0942236920018331, + 1.2164015289999952, + 1.3424603439998464, + 1.4582405809996999, + 1.565261890002148, + 1.6837346329994034, + 1.7918082020005386, + 1.897044643999834, + 2.021567955001956, + 2.151013316000899, + 2.289058919999661, + 2.4211752510018414, + 2.5481206360018405, + 2.674713514999894, + 2.8035346990000107, + 2.949811602000409, + 3.09554646000106, + 3.2371215050006867, + 3.394748973001697, + 3.527810554998723, + 3.66016935999869, + 3.785835719001625, + 3.8995512450019305, + 4.026999310000974, + 4.154745913001534, + 4.2885806459999, + 4.428103680998902, + 4.566841060001025, + 4.690276241999527, + 4.809191225998802, + 4.927433161999943, + 5.059459913001774, + 5.198580897002103, + 5.335536182999931, + 5.467458955001348, + 5.6049328400004015, + 5.738688925001043, + 5.867466447001789, + 6.003864375001285, + 6.141292872001941, + 6.265982690001692, + 6.3910812309986795, + 6.516905036998651, + 6.616564556999947, + 6.728444762000436, + 6.8435656930014375, + 6.960736739001732, + 7.096564007999405, + 7.239053450000938, + 7.398890413998743, + 7.5417529299993475, + 7.69300203199964, + 7.809392826999101, + 7.920214477999252, + 8.025386117999005, + 8.140618731002178, + 8.265523480000411, + 8.383308406999276, + 8.484383519000403, + 8.605470365000656, + 8.693699751998793, + 8.775515678000374 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 125.745082501453, + "p95": 151.24910200029262, + "max": 161.0905540001113 + } + }, + { + "start": 30286.995481125, + "first": 30287.470676959, + "last": 30296.001377514, + "end": 30296.001471806, + "elapsed_s": 9.005990681002004, + "chunk_offsets_s": [ + 0.4751958340020792, + 0.5442790529996273, + 0.6163167650011019, + 0.6997756660020968, + 0.7950019660020189, + 0.8818733290027012, + 0.980709357001615, + 1.0946595320019696, + 1.2162793099996634, + 1.343452213001001, + 1.458953133002069, + 1.565778815001977, + 1.684381991999544, + 1.7927709640025569, + 1.898105595002562, + 2.0222535400025663, + 2.152101653002319, + 2.289742667002429, + 2.4208617020012753, + 2.5487384219995874, + 2.674589296002523, + 2.8036862020016997, + 2.950067099001899, + 3.0957935949991224, + 3.2372499869998137, + 3.3947861550004745, + 3.5283739719998266, + 3.660711035001441, + 3.786347195000417, + 3.899517221001588, + 4.027168142001756, + 4.154842867999832, + 4.2887543860015285, + 4.428439368002728, + 4.566560822000611, + 4.690575814001932, + 4.809203930002695, + 4.927601818002586, + 5.059848053002497, + 5.199081872000534, + 5.335138582999207, + 5.467324915000063, + 5.605338966001, + 5.738741788001789, + 5.868081202999747, + 6.004199791001156, + 6.1413134560025355, + 6.265913092000119, + 6.391355880001356, + 6.516654417999234, + 6.61701487100072, + 6.728510142002051, + 6.844166494000092, + 6.960871662002319, + 7.096843134000665, + 7.23931788399932, + 7.399011748002522, + 7.54257883000173, + 7.692469142999471, + 7.809431282999867, + 7.920317954001803, + 8.0258156029995, + 8.140747354002087, + 8.26540481399934, + 8.38337464900178, + 8.484052784002415, + 8.605998475002707, + 8.693628168999567, + 8.776307943000575, + 8.844162192999647, + 8.918973648000247, + 9.005896389000554 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 124.65745999725186, + "p95": 146.38089700019918, + "max": 159.69386400320218 + } + }, + { + "start": 30286.995313874, + "first": 30287.089387733, + "last": 30295.601606618, + "end": 30295.601732708, + "elapsed_s": 8.606418834002397, + "chunk_offsets_s": [ + 0.0940738590034016, + 0.5441103320008551, + 0.6172574260017427, + 0.6997174110001652, + 0.7945753960011643, + 0.8817542920005508, + 0.9800159719998192, + 1.0943037990000448, + 1.216563647001749, + 1.3431658610024897, + 1.458488419000787, + 1.5651614289999998, + 1.6840133090008749, + 1.792277455002477, + 1.8972702270002628, + 2.0227791000033903, + 2.151078011000209, + 2.2892770860016753, + 2.420889981000073, + 2.5485955479998665, + 2.674842524000269, + 2.80288433700116, + 2.9497415270016063, + 3.095226432000345, + 3.237659597001766, + 3.395275954000681, + 3.527716020002117, + 3.660462775002088, + 3.785845449001499, + 3.8995266500023718, + 4.027255859000434, + 4.154472418002115, + 4.2890596670004015, + 4.42835704800018, + 4.567431253002724, + 4.690501424000104, + 4.809250000002066, + 4.927488084002107, + 5.0594663780029805, + 5.199320376999822, + 5.335437087000173, + 5.4673265570017975, + 5.605033452000498, + 5.738493060001929, + 5.868308853001508, + 6.004032339002151, + 6.14130955100336, + 6.266255368002021, + 6.391624617001071, + 6.51703782900222, + 6.6170314450027945, + 6.729019386002619, + 6.8440805099999125, + 6.960905117000948, + 7.097699859001295, + 7.239186993003386, + 7.399024116002693, + 7.542050036001456, + 7.692214304002846, + 7.8100164370007406, + 7.920359850002569, + 8.02544175300136, + 8.140550514002825, + 8.26549657400028, + 8.383896910003386, + 8.483940149002592, + 8.606292743999802 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 125.87792450176494, + "p95": 150.16426800139016, + "max": 450.03647299745353 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 132.81416316535515, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 9.69651881900063, + "end_to_end_tps": 132.0061378617448, + "timing_valid": true, + "steady_tps": 151.24390051098644, + "per_stream_tps": [ + 34.4, + 27.4, + 29.7, + 29.9, + 30.0 + ], + "ttft_s": [ + 0.29, + 0.38, + 0.19, + 0.48, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.28548193099777563, + "decode_tps": 34.43197552622206, + "prefill_tps": 49.03988126698317, + "itl_mean_ms": 29.04277157255873, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.92625600055908, + "round_gap_p50_ms": 128.52115799978492, + "round_gap_p95_ms": 152.0339150010841, + "tokens_per_chunk": 4.338983050847458 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3814095149973582, + "decode_tps": 27.375315280047236, + "prefill_tps": 36.70595370463416, + "itl_mean_ms": 36.529259654914725, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.4792179999058, + "round_gap_p50_ms": 121.58393099889508, + "round_gap_p95_ms": 146.41026299796067, + "tokens_per_chunk": 3.0843373493975905 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19021252799939248, + "decode_tps": 29.717609766228076, + "prefill_tps": 73.60188178585543, + "itl_mean_ms": 33.65008181567913, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.79490199906286, + "round_gap_p50_ms": 125.11401499796193, + "round_gap_p95_ms": 150.12081299937563, + "tokens_per_chunk": 3.710144927536232 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.47578197000257205, + "decode_tps": 29.914838655520626, + "prefill_tps": 29.4252428269283, + "itl_mean_ms": 33.42822642352628, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.3339949970832, + "round_gap_p50_ms": 124.89837800239911, + "round_gap_p95_ms": 146.3399960011884, + "tokens_per_chunk": 3.5555555555555554 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09657634700124618, + "decode_tps": 29.987011203717287, + "prefill_tps": 144.9630311635141, + "itl_mean_ms": 33.347771580384666, + "itl_p50_ms": 0.0, + "itl_p95_ms": 139.00694499898236, + "round_gap_p50_ms": 126.31360199884512, + "round_gap_p95_ms": 150.31558799819322, + "tokens_per_chunk": 3.8208955223880596 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 33.19962220941271, + "itl_p50_ms": 0.0, + "itl_p95_ms": 138.79490199906286, + "round_gap_p50_ms": 125.14276900037657, + "round_gap_p95_ms": 150.0702920020558, + "round_gap_mean_ms": 122.69425599130783, + "tokens_per_chunk": 3.657142857142857, + "aggregate_prefill_tps": 147.00748240122155 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "bb1a1d1b1c2f", + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "57a51253f387", + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7a84423c938e", + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "c4f1b0ba0518", + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6149a5740446", + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f" + } + ], + "client_intervals": [ + { + "start": 30296.698230024, + "first": 30296.983711955, + "last": 30304.389618706, + "end": 30304.390111082, + "elapsed_s": 7.691881057999126, + "chunk_offsets_s": [ + 0.28548193099777563, + 0.5451453269997728, + 0.6222394929973234, + 0.7095036339997023, + 0.8054771829993115, + 0.8910494529991411, + 0.9892771300001186, + 1.1038239109984715, + 1.2255529429967282, + 1.344009123997239, + 1.4586115999991307, + 1.5672701359981147, + 1.691398646999005, + 1.7982220429985318, + 1.9032953189998807, + 2.026878068998485, + 2.1561282479997317, + 2.290349035996769, + 2.4170506310001656, + 2.5459666609967826, + 2.677893715997925, + 2.8087195249972865, + 2.952854852999735, + 3.0958681229967624, + 3.236148053998477, + 3.3869189500001085, + 3.5218885839967697, + 3.662072940998769, + 3.7887270439969143, + 3.8999751589981315, + 4.028965723999136, + 4.154397746999166, + 4.2831409749996965, + 4.419741934998456, + 4.558668190999015, + 4.688626777999161, + 4.80940371899851, + 4.924577032998059, + 5.0569842609984335, + 5.197014462999505, + 5.32553562099929, + 5.456860929996765, + 5.599549375998322, + 5.7362619609994, + 5.863451907996932, + 5.99716981599704, + 6.135406588997284, + 6.257807175999915, + 6.380408967997937, + 6.504895279998891, + 6.610501002996898, + 6.72609347999969, + 6.841978355998435, + 6.957967695998377, + 7.094018853997113, + 7.235623191998457, + 7.387657106999541, + 7.534022173000267, + 7.691388682000252 + ], + "chunk_gaps_ms": { + "count": 58, + "p50": 128.6321930001577, + "p95": 152.0339150010841, + "max": 259.66339600199717 + } + }, + { + "start": 30296.697824717, + "first": 30297.079234232, + "last": 30306.394195444, + "end": 30306.394258132, + "elapsed_s": 9.696433414999774, + "chunk_offsets_s": [ + 0.3814095149973582, + 0.5455025759983982, + 0.6230238749994896, + 0.7102047539992782, + 0.8057745279984374, + 0.8919458899981692, + 0.989338453997334, + 1.1042974519987183, + 1.2258813829976134, + 1.3449248910001188, + 1.4589390799992543, + 1.5677317829977255, + 1.6910861859978468, + 1.7987488379985734, + 1.9040830749981978, + 2.027084125998954, + 2.157196362000832, + 2.290274415998283, + 2.4175784819999535, + 2.546534736000467, + 2.678149859999394, + 2.8095704009974725, + 2.953624049998325, + 3.0957387919988832, + 3.2369206889998168, + 3.386990021997917, + 3.5220660689992656, + 3.662636348999513, + 3.7885037069972896, + 3.9004573019992677, + 4.029722420000326, + 4.155119959999865, + 4.283427654998377, + 4.419998957000644, + 4.5594781750005495, + 4.689309069999581, + 4.8100742120004725, + 4.9246429720005835, + 5.056744997000351, + 5.196751278999727, + 5.32614721599748, + 5.456704799998988, + 5.600138812998921, + 5.737313776000519, + 5.863733213998785, + 5.997509221997461, + 6.1356115540002065, + 6.257661658997677, + 6.38070475999848, + 6.505245173000731, + 6.610996144998353, + 6.7268567829996755, + 6.84277571399798, + 6.9579708319979545, + 7.094054987999698, + 7.236454524998408, + 7.388148684000043, + 7.534558946998004, + 7.691914089999045, + 7.808559367000271, + 7.917461285000172, + 8.020110974997806, + 8.133564575997298, + 8.258412961000431, + 8.37202168499789, + 8.471692959999928, + 8.601152925999486, + 8.690014837997296, + 8.771356004999689, + 8.84036352700059, + 8.912731980999524, + 9.000701290999132, + 9.065088567000203, + 9.141795119998278, + 9.211628713997925, + 9.279012943999987, + 9.353399062998506, + 9.429111952998937, + 9.494495937000465, + 9.540699484998186, + 9.592382855000324, + 9.64219517699894, + 9.696370727000613 + ], + "chunk_gaps_ms": { + "count": 82, + "p50": 121.81701799818256, + "p95": 146.41026299796067, + "max": 164.09306100104004 + } + }, + { + "start": 30296.698585196, + "first": 30296.888797724, + "last": 30305.469568587, + "end": 30305.469710127, + "elapsed_s": 8.771124930997757, + "chunk_offsets_s": [ + 0.19021252799939248, + 0.5450157199993555, + 0.6223938039984205, + 0.7092222589999437, + 0.8047827609989326, + 0.890532217999862, + 0.9884750969977176, + 1.1039867769977718, + 1.2249309419967176, + 1.3440035309977247, + 1.4576596199985943, + 1.5675205169973196, + 1.6902671769967128, + 1.7977345439976489, + 1.9030685249999806, + 2.0260273049971147, + 2.155607445998612, + 2.2903336599993054, + 2.416955621996749, + 2.5452459209991503, + 2.6771802179973747, + 2.809177009999985, + 2.9534564329987916, + 3.0956845129985595, + 3.2358395009978267, + 3.3859603139972023, + 3.5218332579970593, + 3.662197761997959, + 3.7878517509998346, + 3.8997537869981898, + 4.02870863499993, + 4.153822649997892, + 4.283038646997738, + 4.419672988999082, + 4.558467890998145, + 4.688806677997491, + 4.808482331998675, + 4.924568742997508, + 5.056891779997386, + 5.195708825998736, + 5.324875963997329, + 5.456171152996831, + 5.599269618000108, + 5.736280563996843, + 5.8630372939987865, + 5.996851638999942, + 6.134994549000112, + 6.257621841999935, + 6.379677717999584, + 6.5043963299976895, + 6.610599369996635, + 6.725576527998783, + 6.842329872997652, + 6.958103377997759, + 7.094047773000057, + 7.235515270996984, + 7.387463123999623, + 7.533587876998354, + 7.690943834997597, + 7.80691914399722, + 7.916077409998252, + 8.019674455998029, + 8.132384384000034, + 8.25723423999807, + 8.371107994997146, + 8.471166734998405, + 8.599581959999341, + 8.688879203997203, + 8.770983390997571 + ], + "chunk_gaps_ms": { + "count": 68, + "p50": 125.38400199991884, + "p95": 150.12081299937563, + "max": 354.803191999963 + } + }, + { + "start": 30296.698123582, + "first": 30297.173905552, + "last": 30305.69810329, + "end": 30305.698238914, + "elapsed_s": 9.00011533200086, + "chunk_offsets_s": [ + 0.47578197000257205, + 0.5455299800014473, + 0.6225431860002573, + 0.7099966480018338, + 0.8050055090025126, + 0.8919124140011263, + 0.989556298001844, + 1.104260140000406, + 1.2253474880017166, + 1.3443746090015338, + 1.458199204000266, + 1.5678000990010332, + 1.6909120220007026, + 1.798695970002882, + 1.903045003000443, + 2.026893527003267, + 2.1571334400032356, + 2.2900273660015955, + 2.4173246530008328, + 2.545885139003076, + 2.677802554000664, + 2.8092108640012157, + 2.953743656002189, + 3.0956913780028117, + 3.236676182001247, + 3.386746474003303, + 3.521875184000237, + 3.6627072430019325, + 3.78860596800223, + 3.8996732170016912, + 4.029329820001294, + 4.154628493000928, + 4.283658314001514, + 4.419767016002879, + 4.559369467002398, + 4.689167603002716, + 4.809040175001428, + 4.92431162000139, + 5.056378360000963, + 5.196403300000384, + 5.3262220300021, + 5.4571749240021745, + 5.599979807000636, + 5.736162418001186, + 5.863076146000822, + 5.997406829003012, + 6.135740824000095, + 6.257454339000105, + 6.380173225999897, + 6.505071604002296, + 6.611197295002057, + 6.726114116001554, + 6.841743267999846, + 6.957649137002591, + 7.093719989999954, + 7.236574508002377, + 7.387649173000682, + 7.53398916900187, + 7.692406284000754, + 7.807746479000343, + 7.916051685002458, + 8.019200034003006, + 8.133048372001213, + 8.25819114100159, + 8.371355819002929, + 8.471461900000577, + 8.599987696001335, + 8.689469280001504, + 8.771118533000845, + 8.84015777500099, + 8.912525094001467, + 8.999979708001774 + ], + "chunk_gaps_ms": { + "count": 71, + "p50": 124.89837800239911, + "p95": 146.3399960011884, + "max": 158.41711499888333 + } + }, + { + "start": 30296.697739313, + "first": 30296.79431566, + "last": 30305.297997413, + "end": 30305.298505602, + "elapsed_s": 8.600766289000603, + "chunk_offsets_s": [ + 0.09657634700124618, + 0.5455166130013822, + 0.6227919630000542, + 0.7099375260004308, + 0.8060153920014272, + 0.8919253300009586, + 0.9895674559993495, + 1.1041093599997112, + 1.2258490399981383, + 1.3444404900001246, + 1.4588651220001339, + 1.5676844759982487, + 1.6915724729988142, + 1.798960446001729, + 1.903177215001051, + 2.0269497359986417, + 2.157117702998221, + 2.2910758170000918, + 2.418035640999733, + 2.5459815860012895, + 2.67830423399937, + 2.8088742600011756, + 2.9533835660004115, + 3.096578150998539, + 3.2365335670001514, + 3.3868491549983446, + 3.522424571998272, + 3.6629574239996145, + 3.7892710259984597, + 3.9002148830004444, + 4.029517490998842, + 4.15476783199847, + 4.283683022000332, + 4.420477512001526, + 4.559484457000508, + 4.689211960001558, + 4.80900819100134, + 4.9248302320011135, + 5.057152598001267, + 5.196357437998813, + 5.325686658001359, + 5.45657430600113, + 5.599945056001161, + 5.7362357120000524, + 5.863676487999328, + 5.997507380998286, + 6.135564611999143, + 6.2580882080001174, + 6.380442830999527, + 6.505156630999409, + 6.611232952000137, + 6.726224275000277, + 6.842802634000691, + 6.957871866001369, + 7.094577225998364, + 7.235912104999443, + 7.387984134998987, + 7.534577138001623, + 7.692705434001255, + 7.808425210998394, + 7.916760265998164, + 8.019989945001726, + 8.133137394001096, + 8.258189608000976, + 8.372171983999579, + 8.47158516700074, + 8.600258099999337 + ], + "chunk_gaps_ms": { + "count": 66, + "p50": 126.63671299924317, + "p95": 150.31558799819322, + "max": 448.94026600013603 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "bb1a1d1b1c2fc14509c5d22591e4db105bcfcc6b8077d1dc71a87d0263c86c05", + "output_sha256": "3c959b006cada3032e471210c80923609a915823bfb3db0fd99126908a57e0d4", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "57a51253f387f040b705d1c94e3f37bc83d66daf00654942db6c9e53bec52f7c", + "output_sha256": "cede9c0aa91c1ebd0dd16f90f6cc2f4613adde919c4491a458508d51a2287566", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7a84423c938e631c6a2b58891e9ef35039d80ab47a97a6b3cdc383caf7297811", + "output_sha256": "06bc4587a007055736033f08b4a8cfe55c837b59356f7fafeaecb386563bf191", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "c4f1b0ba05181a859f362ede56d199202e852d7d5305afb698dd5e67bbdf582b", + "output_sha256": "c17225bdea987d85506074fab1d8bc7f8b60d0034aef37ad743813941410b194", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6149a57404465b949c07352142d48269ed6b238bdef99ca2f42c000202726802", + "output_sha256": "67d3d65af4d487486531a3b96dc003fc7a2cbfc62bfadfad014e7e83e35d808f", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.4, + "per_stream_tps": 42.4, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.084, + "decode_tps": 42.44, + "prefill_tps": null, + "itl_mean_ms": 23.563, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.514, + "round_gap_p50_ms": 53.518, + "round_gap_p95_ms": 72.7, + "tokens_per_chunk": 2.349 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 23.563, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.514, + "round_gap_p50_ms": 53.518, + "round_gap_p95_ms": 72.7, + "tokens_per_chunk": 2.349 + }, + "timing_valid": true, + "end_to_end_tps": 42.0251606994553, + "makespan_s": 6.0915888419986, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.44014862478598, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.0915888419986, + "end_to_end_tps": 42.0251606994553, + "timing_valid": true, + "per_stream_tps": [ + 42.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08279381299871602, + "decode_tps": 42.44014862478598, + "prefill_tps": null, + "itl_mean_ms": 23.5625942039227, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.94933100152411, + "round_gap_p50_ms": 53.517585998633876, + "round_gap_p95_ms": 72.70023099772516, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.5625942039227, + "itl_p50_ms": 0.0, + "itl_p95_ms": 66.94933100152411, + "round_gap_p50_ms": 53.517585998633876, + "round_gap_p95_ms": 72.70023099772516, + "round_gap_mean_ms": 55.633902981484155, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 30387.689444656, + "first": 30387.772238469, + "last": 30393.780699991, + "end": 30393.781033498, + "elapsed_s": 6.0915888419986, + "chunk_offsets_s": [ + 0.08279381299871602, + 0.1398717049996776, + 0.19921745499959798, + 0.25526793400058523, + 0.3081566649998422, + 0.3558845330007898, + 0.4063301460009825, + 0.45318328099892824, + 0.5083549010014394, + 0.5691073369998776, + 0.6356033109987038, + 0.6830580070018186, + 0.743627921998268, + 0.8022970669990173, + 0.8526532750001934, + 0.9008551129991247, + 0.9481127690014546, + 0.9930691139998089, + 1.0371974369991221, + 1.0849768910011335, + 1.1332316510015517, + 1.1861809180009004, + 1.2426333889998205, + 1.2931386400014162, + 1.3445973950001644, + 1.402176072999282, + 1.4603070449993538, + 1.5202179459993204, + 1.5890264359986759, + 1.6458794960017258, + 1.700084350999532, + 1.7493561919982312, + 1.8005447020004794, + 1.8560306680010399, + 1.914355737000733, + 1.9723150620011438, + 2.030332530001033, + 2.103032760998758, + 2.1722858089997317, + 2.2202771679985744, + 2.2726900439993187, + 2.320237031999568, + 2.365217221999046, + 2.4142128089988546, + 2.4669128829991678, + 2.52308056900074, + 2.5771880989996134, + 2.645001324999612, + 2.706900888999371, + 2.7561035270009597, + 2.8031271639993065, + 2.8606327140005305, + 2.9315310979982314, + 2.993888977998722, + 3.0467449630014016, + 3.098837016001198, + 3.151875897998252, + 3.21140288999959, + 3.2678136190006626, + 3.3169403680003597, + 3.3634599889992387, + 3.416038028000912, + 3.464299060000485, + 3.5101185659987095, + 3.5672033119990374, + 3.626180375998956, + 3.6729833719982707, + 3.7229023600011715, + 3.7703878809988964, + 3.82926343500003, + 3.8848602729995036, + 3.9305168469982164, + 3.9757055750014842, + 4.024849690998963, + 4.085121971998888, + 4.168110783000884, + 4.250818676999188, + 4.327285529998335, + 4.394234860999859, + 4.442647121999471, + 4.490298070999415, + 4.536479996000708, + 4.597331852000934, + 4.673407710000902, + 4.7413893369994184, + 4.796137402001477, + 4.840384487000847, + 4.90038693599854, + 4.9650237370005925, + 5.026398737001728, + 5.086353965001763, + 5.133156678999512, + 5.182639441001811, + 5.229692936998617, + 5.2764571879997675, + 5.323725155001739, + 5.372382575998927, + 5.435118069999589, + 5.502900855000917, + 5.554321258001437, + 5.607823892001761, + 5.658865997000248, + 5.721852245998889, + 5.799475834999612, + 5.863182189001236, + 5.925252378998266, + 5.990997403998335, + 6.037737749000371, + 6.091255334999005 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 53.81255799875362, + "p95": 72.70023099772516, + "max": 82.98881100199651 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.455267940924855, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.091545770999801, + "end_to_end_tps": 42.02545784335179, + "timing_valid": true, + "per_stream_tps": [ + 42.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08463837799718021, + "decode_tps": 42.455267940924855, + "prefill_tps": null, + "itl_mean_ms": 23.554203011778608, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.51424399731332, + "round_gap_p50_ms": 54.35342400232912, + "round_gap_p95_ms": 73.47709000168834, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.554203011778608, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.51424399731332, + "round_gap_p50_ms": 54.35342400232912, + "round_gap_p95_ms": 73.47709000168834, + "round_gap_mean_ms": 55.61409044447727, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 30393.782038786, + "first": 30393.866677164, + "last": 30399.872998932, + "end": 30399.873584557, + "elapsed_s": 6.091545770999801, + "chunk_offsets_s": [ + 0.08463837799718021, + 0.14215422499910346, + 0.2010749119981483, + 0.2569985859990993, + 0.30940829099927214, + 0.3574203659991326, + 0.4102853820004384, + 0.45816234899757546, + 0.5125157729999046, + 0.578376768000453, + 0.6433225500004482, + 0.6885897899992415, + 0.7489290359990264, + 0.8063609469973017, + 0.8568971339991549, + 0.9049378959971364, + 0.9519347200002812, + 0.9964943579980172, + 1.0410334849984793, + 1.0901636659982614, + 1.1387791449997167, + 1.1884266979977838, + 1.2428407499974128, + 1.2924214140002732, + 1.3433637579983042, + 1.3993758859978698, + 1.4584523679986887, + 1.5218600579974009, + 1.5953371479990892, + 1.6490709600002447, + 1.701275152998278, + 1.7482735909979965, + 1.798896418997174, + 1.8541823319974355, + 1.9138391909982602, + 1.9702320649994363, + 2.0269347709981957, + 2.100679686998774, + 2.16700598999887, + 2.214373450999119, + 2.265420059000462, + 2.3125979459982773, + 2.357141649998084, + 2.408729601000232, + 2.4618332119971456, + 2.5208208419971925, + 2.579421282000112, + 2.6443035819975194, + 2.704533520998666, + 2.755039872998168, + 2.8013580669976363, + 2.857549222000671, + 2.9289037700000335, + 2.9908202169972355, + 3.0441811139971833, + 3.0987146769984975, + 3.1517117919975135, + 3.2099417919998814, + 3.26762259699899, + 3.317425152999931, + 3.3625374609982828, + 3.415275969000504, + 3.4644893349977792, + 3.5120500609991723, + 3.5755381179988035, + 3.630907787999604, + 3.6762542689975817, + 3.7232850679974945, + 3.771331473999453, + 3.828863401999115, + 3.885879412999202, + 3.9306404140006634, + 3.976743285998964, + 4.025686200999189, + 4.086679252999602, + 4.169883841997944, + 4.251981402998354, + 4.329195284997695, + 4.3950229229994875, + 4.444122287997743, + 4.489786474998255, + 4.5406511219989625, + 4.6055919299979, + 4.678228936998494, + 4.742716893997567, + 4.797542026997689, + 4.841408852000313, + 4.902037116000429, + 4.966399287997774, + 5.029368536997936, + 5.091048940998007, + 5.138552237000113, + 5.186235048997332, + 5.233368089000578, + 5.279113807999238, + 5.325773815999128, + 5.373871067997243, + 5.435595031998673, + 5.503329666000354, + 5.55870524599959, + 5.613392522998765, + 5.660825233997457, + 5.720234893000452, + 5.798454700998263, + 5.862340783998661, + 5.92506043100002, + 5.990574674997333, + 6.0373995489972, + 6.090960146000725 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 54.383738000979065, + "p95": 73.47709000168834, + "max": 83.20458899834193 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.43467893374513, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.093732240999088, + "end_to_end_tps": 42.01037884100203, + "timing_valid": true, + "per_stream_tps": [ + 42.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08421000599992112, + "decode_tps": 42.43467893374513, + "prefill_tps": null, + "itl_mean_ms": 23.565631345092484, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.1758450003399, + "round_gap_p50_ms": 53.39313799777301, + "round_gap_p95_ms": 71.81436999962898, + "tokens_per_chunk": 2.3486238532110093 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.565631345092484, + "itl_p50_ms": 0.0, + "itl_p95_ms": 65.1758450003399, + "round_gap_p50_ms": 53.39313799777301, + "round_gap_p95_ms": 71.81436999962898, + "round_gap_mean_ms": 55.64107400924614, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + } + ], + "client_intervals": [ + { + "start": 30399.874843965, + "first": 30399.959053971, + "last": 30405.968289964, + "end": 30405.968576206, + "elapsed_s": 6.093732240999088, + "chunk_offsets_s": [ + 0.08421000599992112, + 0.14169242899879464, + 0.20026214799872832, + 0.2569916309985274, + 0.3095524349992047, + 0.3594939169997815, + 0.4106693819994689, + 0.4632867139989685, + 0.5188936239974282, + 0.5799031479982659, + 0.641415527999925, + 0.686974738000572, + 0.7473117740009911, + 0.8059682290004275, + 0.8571710219985107, + 0.904560857998149, + 0.9521960979982396, + 0.9987889010008075, + 1.044228283000848, + 1.0918718920001993, + 1.140803534999577, + 1.1903886849977425, + 1.244989374998113, + 1.2955932209988532, + 1.3463753680007358, + 1.404020567999396, + 1.4668863769984455, + 1.53033343000061, + 1.5978040150002926, + 1.6507285679981578, + 1.7011301909988106, + 1.7490046469974914, + 1.8007248469984916, + 1.8554478529986227, + 1.9155184080009349, + 1.9735038859980705, + 2.03238484999747, + 2.1046197389987356, + 2.172728253000969, + 2.2197738219983876, + 2.2725409579979896, + 2.321618354999373, + 2.367369300998689, + 2.4187877949989343, + 2.4761702329997206, + 2.5336895549990004, + 2.587646764000965, + 2.649295530998643, + 2.7078278089975356, + 2.7569771819980815, + 2.803358720997494, + 2.8610510180005804, + 2.9312334839996765, + 2.9945518729982723, + 3.0473418580004363, + 3.099930755000969, + 3.1530240760002926, + 3.2114281409994874, + 3.2666731129975233, + 3.316803040997911, + 3.361828813998727, + 3.4165315280006325, + 3.4699246659984055, + 3.5186301059984544, + 3.5761189999975613, + 3.631722460999299, + 3.676247120998596, + 3.7229686709979433, + 3.7691276379991905, + 3.827343359000224, + 3.883966990000772, + 3.928678599997511, + 3.974048732998199, + 4.022995488998276, + 4.0832137020006485, + 4.165529826997954, + 4.247350801997527, + 4.325740451000456, + 4.392716800997732, + 4.445601162999083, + 4.494942245000857, + 4.541324840000016, + 4.6020855189999565, + 4.6738998889995855, + 4.739075733999925, + 4.792404394000187, + 4.8385604430004605, + 4.898441515000741, + 4.963420569998561, + 5.0249743339991255, + 5.085313427000074, + 5.133058075000008, + 5.181405013998301, + 5.229066635998606, + 5.275392245999683, + 5.323206279997976, + 5.37148803799937, + 5.4358779579997645, + 5.507677038000111, + 5.560061403000873, + 5.613550595000561, + 5.661757306999789, + 5.721737191997818, + 5.7998966819977795, + 5.863829432000784, + 5.9259068379979, + 5.993131372000789, + 6.041286265997769, + 6.093445998998504 + ], + "chunk_gaps_ms": { + "count": 108, + "p50": 53.44116499873053, + "p95": 71.81436999962898, + "max": 82.31612499730545 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 65.3, + "per_stream_tps": 33.3, + "ttft_s_max": 0.16, + "request": { + "ttft_s": 0.122, + "decode_tps": 33.301, + "prefill_tps": null, + "itl_mean_ms": 30.032, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.255, + "round_gap_p50_ms": 69.77, + "round_gap_p95_ms": 86.879, + "tokens_per_chunk": 2.349 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 30.028, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.062, + "round_gap_p50_ms": 69.632, + "round_gap_p95_ms": 87.45, + "tokens_per_chunk": 2.349 + }, + "timing_valid": true, + "end_to_end_tps": 64.85452971221304, + "makespan_s": 7.894591206997575, + "steady_tps": 66.7, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 65.25542793392611, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 7.897098602999904, + "end_to_end_tps": 64.83393784718662, + "timing_valid": true, + "steady_tps": 67.302057833758, + "per_stream_tps": [ + 33.0, + 33.6 + ], + "ttft_s": [ + 0.16, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16371959399839398, + "decode_tps": 32.97475963317043, + "prefill_tps": null, + "itl_mean_ms": 30.3262256078454, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.02475199868786, + "round_gap_p50_ms": 69.38049500240595, + "round_gap_p95_ms": 85.41020100165042, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08155530899966834, + "decode_tps": 33.61290578162793, + "prefill_tps": null, + "itl_mean_ms": 29.75047758431459, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.91980000265175, + "round_gap_p50_ms": 70.09031199777382, + "round_gap_p95_ms": 86.55459600049653, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 30.03835159608, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.91980000265175, + "round_gap_p50_ms": 69.63168500078609, + "round_gap_p95_ms": 86.48762400116539, + "round_gap_mean_ms": 70.92388571296667, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 30405.969490324, + "first": 30406.133209918, + "last": 30413.866397448, + "end": 30413.866498929, + "elapsed_s": 7.897008604999428, + "chunk_offsets_s": [ + 0.16371959399839398, + 0.22469945599732455, + 0.30536058799771126, + 0.3952569309985847, + 0.48066713200023514, + 0.5493273679967388, + 0.6086119589999726, + 0.6666537659984897, + 0.7290596709972306, + 0.7958120819967007, + 0.8671142909988703, + 0.9293547380002565, + 0.9972813409985974, + 1.0626099859982787, + 1.1250919159974728, + 1.182952942999691, + 1.2374946679992718, + 1.291303837999294, + 1.3497084519985947, + 1.4094190819996584, + 1.4698697019994142, + 1.5312970630002383, + 1.5969698059998336, + 1.6615896909970616, + 1.723433526000008, + 1.7935475279991806, + 1.8650555989988788, + 1.9302766609980608, + 2.0058018949966936, + 2.086727350997535, + 2.1561805099991034, + 2.2246336139978666, + 2.287053020998428, + 2.361050345996773, + 2.4462660909994156, + 2.531330462999904, + 2.6042723699974886, + 2.676068460998067, + 2.7545842219988117, + 2.8165750659973128, + 2.8770120619992667, + 2.95422637899901, + 3.0381397409983038, + 3.124627364999469, + 3.2022477939972305, + 3.2751025119978294, + 3.34580072099925, + 3.4256841119968158, + 3.5049805769995146, + 3.5897472449978522, + 3.670686981997278, + 3.749712955999712, + 3.8317944279988296, + 3.9068590589995438, + 3.9813236689988116, + 4.062268841997138, + 4.131649336999544, + 4.21077375099776, + 4.280405435998546, + 4.3421341859975655, + 4.406670864998887, + 4.481877578997228, + 4.544098810998548, + 4.604969888998312, + 4.680358934998367, + 4.7621664500002225, + 4.816955657999642, + 4.880967601999146, + 4.961627426997438, + 5.042198548999295, + 5.115771417997166, + 5.180766006997146, + 5.24588232199676, + 5.307112098998914, + 5.382254473999637, + 5.465829273998679, + 5.554812415997731, + 5.646237671997369, + 5.741681935996894, + 5.818420347997744, + 5.882835672997317, + 5.941833136999776, + 6.006941153998923, + 6.079969626996899, + 6.142426057998819, + 6.217584945999988, + 6.277930885997193, + 6.3379693959977885, + 6.406891208996967, + 6.4841199689981295, + 6.553281666998373, + 6.618695403998572, + 6.676796111998556, + 6.735352425999736, + 6.794326606999675, + 6.855041717997665, + 6.922716921999381, + 7.000636241999018, + 7.074356182998599, + 7.147534739997354, + 7.219276995998371, + 7.298389726998721, + 7.380414478997409, + 7.447185297998658, + 7.509184810998704, + 7.587411546999647, + 7.667647198999475, + 7.730942625999887, + 7.796974744996987, + 7.843743164998159, + 7.896907123998972 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 69.41682700198726, + "p95": 85.41020100165042, + "max": 95.44426399952499 + } + }, + { + "start": 30405.969400326, + "first": 30406.050955635, + "last": 30413.637327419, + "end": 30413.637448185, + "elapsed_s": 7.6680478589987615, + "chunk_offsets_s": [ + 0.08155530899966834, + 0.22468701400066493, + 0.3050521189979918, + 0.39554523499828065, + 0.48042542499752017, + 0.5489449069973489, + 0.608383540999057, + 0.6662324469980376, + 0.7289524499974505, + 0.7956450229976326, + 0.8675267189973965, + 0.9292410889975145, + 0.9972288889985066, + 1.0625284149973595, + 1.1246908899993286, + 1.1827367909972963, + 1.2374425849993713, + 1.2915926379973826, + 1.3499252279980283, + 1.4095895629980078, + 1.4698111420002533, + 1.5310618319999776, + 1.5968556449988682, + 1.661451734998991, + 1.7238992200000212, + 1.7933272650006984, + 1.864882238998689, + 1.930582331999176, + 2.00565469600042, + 2.086522151999816, + 2.1566124639975897, + 2.2244154939980945, + 2.287299332998373, + 2.361398057997576, + 2.446638317000179, + 2.5308841330006544, + 2.6045989569975063, + 2.6763852939984645, + 2.754018036997877, + 2.8160256240007584, + 2.8768577629998617, + 2.954155649000313, + 3.03785896699992, + 3.1244135630004166, + 3.201956065997365, + 3.2749861749980482, + 3.3456281119979394, + 3.426011594998272, + 3.5048832059983397, + 3.5894029549999686, + 3.670963597000082, + 3.749540442997386, + 3.8314602430000377, + 3.9063957299986214, + 3.9810228099995584, + 4.062100693998218, + 4.13141698400068, + 4.210488339998847, + 4.280235991998779, + 4.342460933999973, + 4.406954084999597, + 4.481682804998854, + 4.543926392998401, + 4.604679392999969, + 4.680108479999035, + 4.761526416998095, + 4.817214282997156, + 4.8812308329979714, + 4.9613185719972535, + 5.04255742299938, + 5.11569120599961, + 5.18035717499879, + 5.245618098000705, + 5.307363112999155, + 5.382680591999815, + 5.46638741499919, + 5.554629922997265, + 5.6464190109982155, + 5.740905545997521, + 5.818346067997481, + 5.883326138999109, + 5.9417912239987345, + 6.00680220600043, + 6.079846493998048, + 6.142139221999969, + 6.217317571998137, + 6.2774188309995225, + 6.338180638998892, + 6.406744855998113, + 6.483989543998177, + 6.553074019997439, + 6.618607562999387, + 6.676403286997811, + 6.735280992998014, + 6.794248025998968, + 6.854859512997791, + 6.922073033998458, + 7.000398259999201, + 7.074076143999264, + 7.1471611999986635, + 7.218509975999041, + 7.29883979999795, + 7.380735853999795, + 7.447027943999274, + 7.509016341999086, + 7.587187973000255, + 7.667927092999889 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 70.3661244988325, + "p95": 86.55459600049653, + "max": 143.1317050009966 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 65.31132422231232, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 7.890616142998624, + "end_to_end_tps": 64.88720154690324, + "timing_valid": true, + "steady_tps": 66.69312567829061, + "per_stream_tps": [ + 33.0, + 33.6 + ], + "ttft_s": [ + 0.16, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16047025499938172, + "decode_tps": 32.98964071022687, + "prefill_tps": null, + "itl_mean_ms": 30.312545952948117, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.485042999906, + "round_gap_p50_ms": 68.56531299854396, + "round_gap_p95_ms": 87.20340400032, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08152319899818394, + "decode_tps": 33.64157186968128, + "prefill_tps": null, + "itl_mean_ms": 29.72512710980749, + "itl_p50_ms": 0.0, + "itl_p95_ms": 83.70113300043158, + "round_gap_p50_ms": 70.19157900140272, + "round_gap_p95_ms": 87.48889200069243, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 30.018836531377804, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.485042999906, + "round_gap_p50_ms": 68.94299700070405, + "round_gap_p95_ms": 87.45041700240108, + "round_gap_mean_ms": 70.87780847686426, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 30413.867670352, + "first": 30414.028140607, + "last": 30421.757839825, + "end": 30421.758179753, + "elapsed_s": 7.8905094010006, + "chunk_offsets_s": [ + 0.16047025499938172, + 0.2222706909997214, + 0.30016342800081475, + 0.3869976259993564, + 0.4742010299996764, + 0.5482486059991061, + 0.6080992440001864, + 0.6659761289993185, + 0.7285183829990274, + 0.7947733900000458, + 0.8666231440001866, + 0.9280187520016625, + 0.9965840650002065, + 1.061993868999707, + 1.1223414160012908, + 1.1802616960012529, + 1.2358494899999641, + 1.2896234590007225, + 1.3439484070004255, + 1.4007965689997945, + 1.4647272299989709, + 1.5296178300013707, + 1.5972079160019348, + 1.6637848760001361, + 1.7275645310000982, + 1.7984577720017114, + 1.8703707450004003, + 1.9357741659987369, + 2.0141684859991074, + 2.0953731289991993, + 2.16560481399938, + 2.2304107720010506, + 2.293122080001922, + 2.362455886999669, + 2.4466162610005995, + 2.5361846939995303, + 2.6105424000015773, + 2.683997414998885, + 2.7609234800002014, + 2.824136275001365, + 2.8866581820002466, + 2.962875465000252, + 3.0477757199987536, + 3.1352261370011547, + 3.2134045509992575, + 3.2877773580003122, + 3.3540745800019067, + 3.429482694999024, + 3.5132467850016837, + 3.598883278998983, + 3.681368321998889, + 3.7587855980018503, + 3.840822116999334, + 3.915045101999567, + 3.9891906890006794, + 4.070591708001302, + 4.139021687999048, + 4.2178978720003215, + 4.285670625999046, + 4.3461203730003035, + 4.404826029000105, + 4.482097502001125, + 4.549607822998951, + 4.610155708000093, + 4.68655237799976, + 4.768242678001116, + 4.824538222001138, + 4.887901295001939, + 4.9653624890015635, + 5.0478112820019305, + 5.120857584999612, + 5.183894671001326, + 5.249409849999211, + 5.313388904000021, + 5.382188923002104, + 5.466029151000839, + 5.558096380002098, + 5.651478224001039, + 5.746654979000596, + 5.823337912999705, + 5.886181025001861, + 5.9429815799994685, + 6.007810184000846, + 6.0809337419996154, + 6.144643706000352, + 6.220311322998896, + 6.2805473640000855, + 6.339838333999069, + 6.401136614000279, + 6.481624702999397, + 6.552289883002231, + 6.6175786220010195, + 6.677247898001951, + 6.738834774001589, + 6.795351217999269, + 6.854341039001156, + 6.917259543999535, + 6.998474364001595, + 7.073427186998742, + 7.145045337001648, + 7.216484433000005, + 7.294866756001284, + 7.36913566200019, + 7.432468607999908, + 7.5002355510005145, + 7.581039713000791, + 7.661864745001367, + 7.72536369700174, + 7.79132870399917, + 7.837721457999578, + 7.890169473001151 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 68.68266600031347, + "p95": 87.20340400032, + "max": 95.17675499955658 + } + }, + { + "start": 30413.86756361, + "first": 30413.949086809, + "last": 30421.528994222, + "end": 30421.529160868, + "elapsed_s": 7.661597258000256, + "chunk_offsets_s": [ + 0.08152319899818394, + 0.22216436499729753, + 0.3001758209975378, + 0.3869888310000533, + 0.4744101319993206, + 0.5486460689971864, + 0.6085084279984585, + 0.6662057639987324, + 0.7285069979989203, + 0.795250456998474, + 0.8668185739988985, + 0.9283336359985697, + 0.996968958999787, + 1.06179181999687, + 1.1221775859994523, + 1.1799700149967975, + 1.2357529509972665, + 1.2895664349998697, + 1.3439339039978222, + 1.401185524999164, + 1.4645670779973443, + 1.5295093779968738, + 1.5970435419985733, + 1.663615719997324, + 1.7281186289983452, + 1.798726553999586, + 1.870150177997857, + 1.9356221529997129, + 2.0140698359973612, + 2.0951068109970947, + 2.1652983899984974, + 2.23069234599825, + 2.2933284469982027, + 2.3622714439989068, + 2.4466769809987454, + 2.5365654139968683, + 2.610365152999293, + 2.68392377499913, + 2.7613086299970746, + 2.8239665269975376, + 2.8866342909968807, + 2.962734245997126, + 3.04778550699848, + 3.1352743989991723, + 3.213758597998094, + 3.2881096089986386, + 3.353958132000116, + 3.429502187998878, + 3.5132033209993097, + 3.5993113499971514, + 3.681192721996922, + 3.7587158599999384, + 3.8405993589985883, + 3.9148436919967935, + 3.9887805609978386, + 4.070891688999836, + 4.138830143998348, + 4.217801172999316, + 4.28614526399906, + 4.346048923998751, + 4.4046848739999405, + 4.482461655999941, + 4.54994333599825, + 4.610079861999111, + 4.686716703999991, + 4.768024396998953, + 4.824938259996998, + 4.887790236000001, + 4.965340005997859, + 5.047682135998912, + 5.120754296996893, + 5.183823652998399, + 5.249250272998324, + 5.313406926998141, + 5.382112883999071, + 5.466414397997141, + 5.558289698998124, + 5.651420414997119, + 5.746322055998462, + 5.823766511999565, + 5.886574218999158, + 5.943273053999292, + 6.007777689999784, + 6.080799702998775, + 6.145185763998597, + 6.220247101999121, + 6.280308756999148, + 6.340490667997074, + 6.40147056099886, + 6.481423707999056, + 6.552222926999093, + 6.617570449998311, + 6.6776020739998785, + 6.738617627997883, + 6.795327150997764, + 6.8546696129997144, + 6.917189885996777, + 6.998665459999756, + 7.073124821999954, + 7.145018439998239, + 7.216198043999611, + 7.295188917996711, + 7.369555369998125, + 7.432425895000051, + 7.500230192999879, + 7.5810505709996505, + 7.661430611999094 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 70.39975200132176, + "p95": 87.48889200069243, + "max": 140.64116599911358 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 65.29244186864975, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 7.894591206997575, + "end_to_end_tps": 64.85452971221304, + "timing_valid": true, + "steady_tps": 66.68853115958743, + "per_stream_tps": [ + 32.6, + 34.0 + ], + "ttft_s": [ + 0.08, + 0.16 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08336746499844594, + "decode_tps": 32.64622093432487, + "prefill_tps": null, + "itl_mean_ms": 30.631416788231697, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.6431280001998, + "round_gap_p50_ms": 69.784424998943, + "round_gap_p95_ms": 88.06609900057083, + "tokens_per_chunk": 2.3063063063063063 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16277979399819742, + "decode_tps": 33.98572968663734, + "prefill_tps": null, + "itl_mean_ms": 29.424114450989247, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.72154300336842, + "round_gap_p50_ms": 69.75463400158333, + "round_gap_p95_ms": 86.23743800126249, + "tokens_per_chunk": 2.392523364485981 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 30.02776561961047, + "itl_p50_ms": 0.0, + "itl_p95_ms": 82.06209600029979, + "round_gap_p50_ms": 69.784424998943, + "round_gap_p95_ms": 87.60879400142585, + "round_gap_mean_ms": 70.8988910463025, + "tokens_per_chunk": 2.3486238532110093, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + } + ], + "client_intervals": [ + { + "start": 30421.759135427, + "first": 30421.842502892, + "last": 30429.653514173, + "end": 30429.653628767, + "elapsed_s": 7.894493339998007, + "chunk_offsets_s": [ + 0.08336746499844594, + 0.22225746099866228, + 0.30269037399921217, + 0.3866547180005, + 0.47170290799840586, + 0.5433035629976075, + 0.6073370829981286, + 0.6669749829998182, + 0.7293223609995039, + 0.7961594509979477, + 0.8665397740005574, + 0.9281504789978499, + 0.9967508539994014, + 1.0619330230001651, + 1.124835547998373, + 1.182587371000409, + 1.2379085079992365, + 1.293663298998581, + 1.3470546959979401, + 1.403682629999821, + 1.46278316399912, + 1.5236851869995007, + 1.5957115780001914, + 1.6652927789982641, + 1.7277511280008184, + 1.7970589729993662, + 1.8692551599997387, + 1.9326486840000143, + 2.009320705998107, + 2.0915953389994684, + 2.161704138998175, + 2.2262167139997473, + 2.2873763009993127, + 2.3552547699982824, + 2.4358874860008655, + 2.519139426000038, + 2.598496750000777, + 2.675612661998457, + 2.7531194019975374, + 2.8167249469988747, + 2.878686611999001, + 2.955395878998388, + 3.0385624169975927, + 3.1266285159981635, + 3.20546652199846, + 3.277055560000008, + 3.343695106999803, + 3.4199001179986226, + 3.4981438529976003, + 3.5862511140003335, + 3.6724632209989068, + 3.7501516880001873, + 3.8322017370010144, + 3.9060167569987243, + 3.980068017997837, + 4.061549417998322, + 4.132710617999692, + 4.211972556000546, + 4.2810901689990715, + 4.338889194998046, + 4.397922131000087, + 4.473843693998788, + 4.536882231001073, + 4.603985138001008, + 4.682069582999247, + 4.763528384999518, + 4.819392277000588, + 4.883506923000823, + 4.962705502999597, + 5.044266277000133, + 5.118776212999364, + 5.183552492999297, + 5.248395266000443, + 5.312556261000282, + 5.381712647998938, + 5.461963269000989, + 5.5514063499977055, + 5.650073074000829, + 5.745733191997715, + 5.821817831998487, + 5.885255248998874, + 5.944477910998103, + 6.007508396000048, + 6.079660127998068, + 6.144759488997806, + 6.220720065997739, + 6.281977571998141, + 6.34076434400049, + 6.401972740997735, + 6.477651373999834, + 6.547435798998777, + 6.617376604997844, + 6.678948343000229, + 6.739274749998003, + 6.7981354980001925, + 6.856704306999745, + 6.921243312997831, + 6.998413368000911, + 7.070861921998585, + 7.14591875699989, + 7.2168964379998215, + 7.297732471997733, + 7.373388136998983, + 7.436671455998294, + 7.499035572000139, + 7.581678700000339, + 7.666486231999443, + 7.729035737000231, + 7.794033111000317, + 7.840640682999947, + 7.894378745997528 + ], + "chunk_gaps_ms": { + "count": 110, + "p50": 69.86261549900519, + "p95": 88.06609900057083, + "max": 138.88999600021634 + } + }, + { + "start": 30421.75903756, + "first": 30421.921817354, + "last": 30429.424966539, + "end": 30429.425097317, + "elapsed_s": 7.666059756997129, + "chunk_offsets_s": [ + 0.16277979399819742, + 0.22268602899930556, + 0.30288521200054674, + 0.38708402200063574, + 0.47167193299901555, + 0.5433352899999591, + 0.60734370299906, + 0.6671741239988478, + 0.7293775949983683, + 0.7964525079987652, + 0.8665367840003455, + 0.9285159289975127, + 0.9972080779989483, + 1.0621442209994711, + 1.1255236879987933, + 1.182900976997189, + 1.2383294319988636, + 1.2942347019998124, + 1.3473960949995671, + 1.4041138529973978, + 1.4633216239999456, + 1.5236907039979997, + 1.596089741000469, + 1.6656455640004424, + 1.7281256450005458, + 1.797421448998648, + 1.869522902998142, + 1.9331678269991244, + 2.0101792979985476, + 2.0918464669994137, + 2.162372497998149, + 2.2263766119976935, + 2.2875678290001815, + 2.3555128070001956, + 2.4357427159993676, + 2.5194023240001115, + 2.598682537998684, + 2.675937238000188, + 2.753287614999863, + 2.817269260998728, + 2.879248100998666, + 2.9557766809994064, + 3.039283195997996, + 3.1269170159976056, + 3.205757020998135, + 3.2776944629986247, + 3.3441339259989036, + 3.4201302980000037, + 3.4985190739971586, + 3.5861278679985844, + 3.672365305999847, + 3.7506336829974316, + 3.8318863860004058, + 3.9064236729973345, + 3.9800687369970547, + 4.061790280000423, + 4.133095737997792, + 4.212284596000245, + 4.281438349000382, + 4.339100648998283, + 4.3981463300006, + 4.474100578998332, + 4.536894687997119, + 4.603980324998702, + 4.682279533997644, + 4.763921228997788, + 4.8193701449999935, + 4.883833210999001, + 4.9632272059971, + 5.044546350000019, + 5.119275831999403, + 5.18397289099812, + 5.248785438998311, + 5.312902138997742, + 5.382014372997219, + 5.462152542997501, + 5.5514455449992965, + 5.650365011999384, + 5.746156291999796, + 5.821713137000188, + 5.88540257799832, + 5.944835206999414, + 6.007873030997871, + 6.079599167998822, + 6.145094731997233, + 6.220532933999493, + 6.2818954079994, + 6.340993451998656, + 6.402124451000418, + 6.477831707998121, + 6.547586341999704, + 6.61723296999844, + 6.678802004997124, + 6.739528774000064, + 6.79845692500021, + 6.856641731999844, + 6.921615736999229, + 6.998770775000594, + 7.071112219000497, + 7.1462486749987875, + 7.217369879999751, + 7.298026521999418, + 7.374040489998151, + 7.43725832400014, + 7.499634177998814, + 7.581696273999114, + 7.665928979000455 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 69.91945500158181, + "p95": 86.23743800126249, + "max": 98.91946700008702 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 94.6, + "per_stream_tps": 24.9, + "ttft_s_max": 0.33, + "request": { + "ttft_s": 0.207, + "decode_tps": 24.812, + "prefill_tps": null, + "itl_mean_ms": 40.311, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.242, + "round_gap_p50_ms": 94.772, + "round_gap_p95_ms": 108.935, + "tokens_per_chunk": 2.339 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 40.2, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.287, + "round_gap_p50_ms": 94.741, + "round_gap_p95_ms": 109.149, + "tokens_per_chunk": 2.359 + }, + "timing_valid": true, + "end_to_end_tps": 94.27040789657362, + "makespan_s": 10.862369462996867, + "steady_tps": 99.7, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 94.64792755941666, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.862369462996867, + "end_to_end_tps": 94.27040789657362, + "timing_valid": true, + "steady_tps": 98.95829451943506, + "per_stream_tps": [ + 24.1, + 25.4, + 24.0, + 26.1 + ], + "ttft_s": [ + 0.17, + 0.33, + 0.25, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16607577099784976, + "decode_tps": 24.07626197107097, + "prefill_tps": null, + "itl_mean_ms": 41.53468678823807, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.67248700276832, + "round_gap_p50_ms": 94.36333999838098, + "round_gap_p95_ms": 109.17419900215464, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3293753629986895, + "decode_tps": 25.380583479096963, + "prefill_tps": null, + "itl_mean_ms": 39.40019743137835, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.02761500133784, + "round_gap_p50_ms": 95.09166900170385, + "round_gap_p95_ms": 107.52815500018187, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.24965973700091126, + "decode_tps": 24.028756960485712, + "prefill_tps": null, + "itl_mean_ms": 41.616801137256424, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.30357299972093, + "round_gap_p50_ms": 94.37422199698631, + "round_gap_p95_ms": 109.18648100050632, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08551454499684041, + "decode_tps": 26.143894186899026, + "prefill_tps": null, + "itl_mean_ms": 38.24984881177764, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.30488399995374, + "round_gap_p50_ms": 94.87224999975297, + "round_gap_p95_ms": 109.1493489984714, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 40.20038354216262, + "itl_p50_ms": 0.0, + "itl_p95_ms": 103.20027499983553, + "round_gap_p50_ms": 94.74078400307917, + "round_gap_p95_ms": 109.1493489984714, + "round_gap_mean_ms": 95.3590493325718, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 30429.655325807, + "first": 30429.821401578, + "last": 30440.412746709, + "end": 30440.412858986, + "elapsed_s": 10.757533178999438, + "chunk_offsets_s": [ + 0.16607577099784976, + 0.4153326159976132, + 0.5209823099976347, + 0.6322977639974852, + 0.7381522759969812, + 0.8308463459979976, + 0.910708548999537, + 1.0058110229983868, + 1.1073213539966673, + 1.1954305439976451, + 1.284512698999606, + 1.3670612999994773, + 1.4534975919996214, + 1.541251722999732, + 1.634763710997504, + 1.7425481769969338, + 1.8421965229972557, + 1.9471042109980772, + 2.0401918190000288, + 2.1231439469993347, + 2.224430971997208, + 2.317653182999493, + 2.401668404996599, + 2.5010588499972073, + 2.593191947999003, + 2.6888682059980056, + 2.791215146997274, + 2.874053206996905, + 2.974271767998289, + 3.0744317849967047, + 3.1707396619967767, + 3.258422838996921, + 3.3471423749979294, + 3.4334522869976354, + 3.5284794399995008, + 3.6298373189965787, + 3.7340935519969207, + 3.8312650619991473, + 3.9313853469975584, + 4.017904106996866, + 4.106223376998969, + 4.200981607998983, + 4.301922548998846, + 4.404457372998877, + 4.497391760996834, + 4.590721423999639, + 4.678003935998277, + 4.775283904997195, + 4.878379773999768, + 4.97970516499845, + 5.076949456997681, + 5.167538156998489, + 5.270483200998569, + 5.373469407997618, + 5.482643606999773, + 5.5969511619987315, + 5.6878360849987075, + 5.788968640998064, + 5.885510287997022, + 5.979747708999639, + 6.0821352589991875, + 6.182159137999406, + 6.279507076997106, + 6.3831795639998745, + 6.48367344199869, + 6.585961770997528, + 6.67675896399669, + 6.767529418997583, + 6.862661133000074, + 6.962413165998441, + 7.058734756999911, + 7.1459535299982235, + 7.244047458996647, + 7.33993414699944, + 7.427599568000005, + 7.524524662996555, + 7.626533369999379, + 7.741462711997883, + 7.860395394000079, + 7.95475873399846, + 8.041424926999753, + 8.12438304799798, + 8.213389411997923, + 8.303247517997079, + 8.381701933998556, + 8.47340729699863, + 8.566808492996643, + 8.64687150899772, + 8.731661735997477, + 8.82172430999708, + 8.910035318996961, + 8.996871529998316, + 9.07722770199689, + 9.166222619998734, + 9.254811797996808, + 9.344455806996848, + 9.442136021996703, + 9.546776360999502, + 9.636259382998105, + 9.737062289997994, + 9.839513263999834, + 9.931115559997124, + 10.01496540199878, + 10.097737544998381, + 10.184639982999215, + 10.280972209999163, + 10.376914404998388, + 10.446286236998276, + 10.519469322996883, + 10.607029219998367, + 10.683051629999682, + 10.757420901998557 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 94.36333999838098, + "p95": 109.17419900215464, + "max": 249.25684499976342 + } + }, + { + "start": 30429.655959483, + "first": 30429.985334846, + "last": 30440.032385191, + "end": 30440.032491791, + "elapsed_s": 10.376532308000606, + "chunk_offsets_s": [ + 0.3293753629986895, + 0.414782670999557, + 0.5205112030016608, + 0.6323340160015505, + 0.7378579380019801, + 0.8304433619996416, + 0.9098447559990746, + 1.0051156199988327, + 1.1064634300018952, + 1.1946662020018266, + 1.2843958500016015, + 1.3662010730004113, + 1.4529502059995139, + 1.5410391170007642, + 1.6344621290008945, + 1.7419902840010764, + 1.842316091999237, + 1.9462967239996942, + 2.0393078769993735, + 2.1221408689998498, + 2.223173966001923, + 2.316648169999098, + 2.401312403999327, + 2.5002551050019974, + 2.5915714250004385, + 2.688499028001388, + 2.790518592999433, + 2.8739190400010557, + 2.9738962420015014, + 3.074062143001356, + 3.169271776001551, + 3.25790391800183, + 3.3462365080013114, + 3.4329305510000268, + 3.5281811530003324, + 3.628828084001725, + 3.7332465819999925, + 3.8310178669999004, + 3.9303839480016904, + 4.016916749002121, + 4.10551118299918, + 4.200602852000884, + 4.300960600001417, + 4.403727157001413, + 4.496562509000796, + 4.589974529000756, + 4.6775995130010415, + 4.774831955000991, + 4.877411683999526, + 4.978784522001661, + 5.076799026999652, + 5.167282933998649, + 5.269763427000726, + 5.373124009998719, + 5.481863113000145, + 5.596541031001834, + 5.68760217900126, + 5.788825630999781, + 5.884782709999854, + 5.979277785001614, + 6.081134307001776, + 6.181616388999828, + 6.278990426999371, + 6.382018042000709, + 6.483262510999339, + 6.585453404000873, + 6.676505260998965, + 6.767198781999468, + 6.862600991000363, + 6.961346233001677, + 7.0575736350001534, + 7.14519244999974, + 7.243616570998711, + 7.339698562998819, + 7.426803786001983, + 7.524023476002185, + 7.6255642879987136, + 7.740645070000028, + 7.859093994000432, + 7.953846070999134, + 8.040263867998874, + 8.12350444600088, + 8.212431635998655, + 8.302145835001284, + 8.381271829999605, + 8.472568018001766, + 8.565944795001997, + 8.646038707000116, + 8.73076623200177, + 8.821003815999575, + 8.909226824001962, + 8.996152826999605, + 9.077117202999943, + 9.165680128000531, + 9.25438545200086, + 9.343235115000425, + 9.4418986640012, + 9.545757945001242, + 9.635549044000072, + 9.736274343998957, + 9.838578437000251, + 9.930370119000145, + 10.01461602299969, + 10.097475223999936, + 10.18389871600084, + 10.280500336000841, + 10.37642570800017 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 95.15065100094944, + "p95": 107.52815500018187, + "max": 118.44892400040408 + } + }, + { + "start": 30429.655554756, + "first": 30429.905214493, + "last": 30440.517498783, + "end": 30440.517572568, + "elapsed_s": 10.862017811999976, + "chunk_offsets_s": [ + 0.24965973700091126, + 0.4148967060027644, + 0.5211067390009703, + 0.6322562180030218, + 0.7380757570026617, + 0.8304083570001239, + 0.9109033730019291, + 1.0057184320030501, + 1.1065971980024187, + 1.1948455940000713, + 1.2844106100019417, + 1.3664812430033635, + 1.453611771001306, + 1.541247581000789, + 1.6347587700001895, + 1.7421507580002071, + 1.843033771001501, + 1.947142127002735, + 2.0393176800025685, + 2.123086346000491, + 2.223521427000378, + 2.317514762002247, + 2.4018464380023943, + 2.500745370001823, + 2.592039750001277, + 2.688720589001605, + 2.790563848000602, + 2.874271315002261, + 2.9742193010024494, + 3.0744191680023505, + 3.1697317690013733, + 3.2577875160022813, + 3.346390169001097, + 3.4335436150031455, + 3.527917837000132, + 3.629491441002756, + 3.7334333770013473, + 3.83128706700154, + 3.930551026001922, + 4.017474498003139, + 4.105804305003403, + 4.200855180002691, + 4.301298787002452, + 4.404041021000012, + 4.496900680002, + 4.590291974000138, + 4.6779317510008696, + 4.77515629300251, + 4.877882120003051, + 4.979132735003077, + 5.077144984999904, + 5.167391596001835, + 5.269850068001688, + 5.373153641001409, + 5.482340122001915, + 5.596899318003125, + 5.6882863890023145, + 5.789481408002757, + 5.885137096000108, + 5.979877880003187, + 6.08183618700059, + 6.182108110002446, + 6.279340655000851, + 6.382540930000687, + 6.483967126001517, + 6.585645812003349, + 6.677104413000052, + 6.767506504002085, + 6.863186294001935, + 6.961929585002508, + 7.0582035050028935, + 7.145372241000587, + 7.244243003002339, + 7.340167320002365, + 7.427155917001073, + 7.52412105500116, + 7.626092917002097, + 7.741342713001359, + 7.85985828900084, + 7.954354662000696, + 8.040791793002427, + 8.123836684000707, + 8.212910260001081, + 8.302704352001456, + 8.381722644000547, + 8.473008918001142, + 8.566404117002094, + 8.64654894600244, + 8.731086684001639, + 8.821358314002282, + 8.9095873030019, + 8.996498194002925, + 9.077719569002511, + 9.166139770000882, + 9.254475866000575, + 9.343923036001797, + 9.442504852002457, + 9.546388696002396, + 9.636531256001035, + 9.737116119002167, + 9.839209923000453, + 9.93109382900002, + 10.015263869001501, + 10.097928053000942, + 10.18455980300132, + 10.281184003000817, + 10.37652662900291, + 10.446231867001188, + 10.519147592000081, + 10.607281343000068, + 10.683078849000594, + 10.757877525000367, + 10.810653086002276, + 10.8619440270013 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 94.37422199698631, + "p95": 109.18648100050632, + "max": 165.23696900185314 + } + }, + { + "start": 30429.655203105, + "first": 30429.74071765, + "last": 30439.494429097, + "end": 30439.495037158, + "elapsed_s": 9.839834053000232, + "chunk_offsets_s": [ + 0.08551454499684041, + 0.41578966599990963, + 0.5214074100003927, + 0.6328324049973162, + 0.7383226639976783, + 0.8312715409992961, + 0.9105192579991126, + 1.0057723709978745, + 1.1067877519999456, + 1.1953046439994068, + 1.2845483599994623, + 1.366568171997642, + 1.4543337039976905, + 1.5417420199992193, + 1.6352940340002533, + 1.7425795509989257, + 1.8428744479970192, + 1.946727500999259, + 2.0398618189974513, + 2.1227019279976957, + 2.2241807539976435, + 2.317126088997611, + 2.401559902998997, + 2.500837304000015, + 2.592150361997483, + 2.6891388129988627, + 2.7911368850000144, + 2.873883729997033, + 2.974305669999012, + 3.074508910998702, + 3.168805847999465, + 3.2582958189996134, + 3.346506617999694, + 3.4334558049995394, + 3.5281772329981322, + 3.629726273997221, + 3.733706989998609, + 3.831221820997598, + 3.931218715999421, + 4.017510013996798, + 4.10639477299992, + 4.201267022999673, + 4.301980420998007, + 4.404299904999789, + 4.497125166999467, + 4.59043338599804, + 4.678209392997815, + 4.775251985000068, + 4.87831804599773, + 4.97973593200004, + 5.077431806999812, + 5.167485783997108, + 5.270027747999848, + 5.373332631999801, + 5.482481980998273, + 5.597133879997273, + 5.688037256997632, + 5.78939967399856, + 5.885206400998868, + 5.979746780998539, + 6.081846404998942, + 6.182145496997691, + 6.279506756996852, + 6.3825911409985565, + 6.483650109999871, + 6.58559730900015, + 6.677119634998235, + 6.7675580119976075, + 6.863040305997856, + 6.961779760000354, + 7.058151531997282, + 7.1457753049980965, + 7.2440019299974665, + 7.34019074599928, + 7.427436645997659, + 7.524349187999178, + 7.626383640999848, + 7.741868223998608, + 7.860116741998354, + 7.954656308000267, + 8.04107392899823, + 8.124127215000044, + 8.213008541999443, + 8.302995123998699, + 8.38212193399886, + 8.473271639999439, + 8.566856977999123, + 8.646938463996776, + 8.731734433000383, + 8.821563098998013, + 8.909882919997472, + 8.996694134999416, + 9.07759404999888, + 9.16658341999937, + 9.255093263996969, + 9.344139654000173, + 9.442134486998839, + 9.546455443000013, + 9.636678343998938, + 9.737276190997363, + 9.839225992000138 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 95.06268149925745, + "p95": 109.1493489984714, + "max": 330.2751210030692 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 94.51360615847234, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.875960738998401, + "end_to_end_tps": 94.15260174011102, + "timing_valid": true, + "steady_tps": 99.71801333956927, + "per_stream_tps": [ + 23.9, + 25.3, + 24.0, + 26.3 + ], + "ttft_s": [ + 0.08, + 0.33, + 0.25, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08369700500043109, + "decode_tps": 23.877311157025986, + "prefill_tps": null, + "itl_mean_ms": 41.88076259607424, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.62147799989907, + "round_gap_p50_ms": 93.51802300079726, + "round_gap_p95_ms": 109.45682500096154, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3339302200001839, + "decode_tps": 25.34874735031205, + "prefill_tps": null, + "itl_mean_ms": 39.4496811294184, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.19607100266148, + "round_gap_p50_ms": 94.20975799730513, + "round_gap_p95_ms": 108.8409900003171, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25191774199993233, + "decode_tps": 24.004008820839918, + "prefill_tps": null, + "itl_mean_ms": 41.65970807058757, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.89358799895854, + "round_gap_p50_ms": 93.37238099760725, + "round_gap_p95_ms": 109.02946700298344, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16843731100016157, + "decode_tps": 26.320122083575267, + "prefill_tps": null, + "itl_mean_ms": 37.99374474117797, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.63064400028088, + "round_gap_p50_ms": 94.80314999746042, + "round_gap_p95_ms": 109.43409800165682, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 40.245974134314544, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.57124700042186, + "round_gap_p50_ms": 93.98786400197423, + "round_gap_p95_ms": 109.15018600280746, + "round_gap_mean_ms": 95.46719445814148, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 30440.521327341, + "first": 30440.605024346, + "last": 30451.284618808, + "end": 30451.284723601, + "elapsed_s": 10.763396259997535, + "chunk_offsets_s": [ + 0.08369700500043109, + 0.42222866299925954, + 0.5273721640005533, + 0.6360232749975694, + 0.7347676129975298, + 0.8305838099986431, + 0.9140723739983514, + 1.008716183998331, + 1.1092476539997733, + 1.197548607000499, + 1.2865007939981297, + 1.3709831579981255, + 1.4589708159983275, + 1.5478928419979638, + 1.640467618999537, + 1.7427265090009314, + 1.8471392479987117, + 1.953139672998077, + 2.0459255700006906, + 2.128510524999001, + 2.22638884099797, + 2.319039330999658, + 2.402614201000688, + 2.502646334000019, + 2.5926911669994297, + 2.683889084997645, + 2.7827799189981306, + 2.8731982659992354, + 2.9749308110003767, + 3.0757287460000953, + 3.171345090999239, + 3.2580159839999396, + 3.349031637997541, + 3.4365387199977704, + 3.529929631000414, + 3.634551109000313, + 3.7338424650006345, + 3.835612635997677, + 3.93907453599968, + 4.026274165000359, + 4.114603227997577, + 4.210868841997581, + 4.310464120997494, + 4.4145327389996964, + 4.507319884000026, + 4.600837907000823, + 4.682363744999748, + 4.773749199997837, + 4.883358806997421, + 4.985454515997844, + 5.082999161000771, + 5.173908595999819, + 5.273961229999259, + 5.379321099997469, + 5.4887779249984305, + 5.604598170997633, + 5.6917580119989, + 5.789589525000338, + 5.8941996799985645, + 5.987553414997819, + 6.09112971599825, + 6.19138884199856, + 6.285304960998474, + 6.390395308000734, + 6.4914943229996425, + 6.592733519999456, + 6.678833734000364, + 6.766122824999911, + 6.868934950998664, + 6.971629203999328, + 7.067733479998424, + 7.157367845000408, + 7.253740475000086, + 7.348820192997664, + 7.437790866999421, + 7.533579527000256, + 7.636514304998855, + 7.746930559998873, + 7.871120298997994, + 7.966780818998814, + 8.052883485997882, + 8.136159500998474, + 8.22358951500064, + 8.312018398999498, + 8.394207694000215, + 8.48510578999776, + 8.579304096998385, + 8.658595799999603, + 8.737623288998293, + 8.829901138000423, + 8.92248506499891, + 9.010162697999476, + 9.09175070999845, + 9.180847962998087, + 9.266584635999607, + 9.35813245999816, + 9.456028650998633, + 9.560482076998596, + 9.65169874699859, + 9.747485600997607, + 9.856252243000199, + 9.948714879999898, + 10.03346388600039, + 10.114804942000774, + 10.201773450000474, + 10.296798030998616, + 10.394447848997515, + 10.464642661998369, + 10.537389730998257, + 10.621961425000336, + 10.694437309000932, + 10.763291466999362 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 93.51802300079726, + "p95": 109.45682500096154, + "max": 338.53165799882845 + } + }, + { + "start": 30440.522242295, + "first": 30440.856172515, + "last": 30450.915841203, + "end": 30450.916054211, + "elapsed_s": 10.393811916001141, + "chunk_offsets_s": [ + 0.3339302200001839, + 0.4210938249998435, + 0.5271163639990846, + 0.6353834710025694, + 0.734069615999033, + 0.8298918410000624, + 0.9129206489997159, + 1.008500522999384, + 1.109065159002057, + 1.196793663999415, + 1.2860479299997678, + 1.3702164940004877, + 1.458669584000745, + 1.5474724100022286, + 1.6388250670024718, + 1.7420324930026254, + 1.845970092999778, + 1.952640194002015, + 2.045265009001014, + 2.1279174000010244, + 2.225440193000395, + 2.318255156002124, + 2.4016151950017957, + 2.502036402001977, + 2.5921259480019216, + 2.6838317250003456, + 2.7823815230003675, + 2.8728222909994656, + 2.9735382060025586, + 3.075084913001774, + 3.170485628001188, + 3.2567700530016737, + 3.3485788390025846, + 3.4353466970023874, + 3.5295564549996925, + 3.633752526002354, + 3.7333326730004046, + 3.835218239000824, + 3.938797692000662, + 4.025727415999427, + 4.113999294000678, + 4.210290638002334, + 4.310206803002075, + 4.413405018000049, + 4.506933594999282, + 4.600114946999383, + 4.681033569999272, + 4.773072694999428, + 4.881913684999745, + 4.98503378100213, + 5.082284964999417, + 5.173260316001688, + 5.27349427900117, + 5.378561712001101, + 5.488305664999643, + 5.603952739002125, + 5.6910988750023535, + 5.7884751400015375, + 5.893002554999839, + 5.986990419001813, + 6.090478287002043, + 6.190904155999306, + 6.285135772999638, + 6.389851038002234, + 6.490673175001575, + 6.592056120000052, + 6.677512883001327, + 6.765304204000131, + 6.86773445200015, + 6.971433917999093, + 7.066573967000295, + 7.156607417000487, + 7.252642674000526, + 7.3481900170008885, + 7.437065173002338, + 7.532565138000791, + 7.635968739999953, + 7.746489612000005, + 7.870374520000041, + 7.966122305999306, + 8.051607155000966, + 8.13544679199913, + 8.222399874000985, + 8.311744131002342, + 8.393160809002438, + 8.483977428000799, + 8.577953037001862, + 8.657862366002519, + 8.737394742001925, + 8.829247566001868, + 8.92141758300204, + 9.009493694000412, + 9.091090342000825, + 9.179715843001759, + 9.265925739000522, + 9.3566789409997, + 9.45563564600161, + 9.559677608001948, + 9.65101680600128, + 9.746469068999431, + 9.855590659000882, + 9.948040695999225, + 10.032311664999725, + 10.114072484000644, + 10.201142890000483, + 10.296256544999778, + 10.393598908001877 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 94.22068749881873, + "p95": 108.8409900003171, + "max": 123.88490800003638 + } + }, + { + "start": 30440.521977988, + "first": 30440.77389573, + "last": 30451.397121288, + "end": 30451.397190212, + "elapsed_s": 10.875212224000279, + "chunk_offsets_s": [ + 0.25191774199993233, + 0.42077033200257574, + 0.5271110860012413, + 0.6355138969993277, + 0.7337008670001524, + 0.8296927459996368, + 0.9139728740010469, + 1.0083017159995507, + 1.1089629890011565, + 1.1971545439992042, + 1.2861686340002052, + 1.3704239670005336, + 1.4583796250008163, + 1.547288555000705, + 1.6392845670015959, + 1.741836453002179, + 1.8467300410011376, + 1.9522182569999131, + 2.045440211000823, + 2.1275164280013996, + 2.225573178002378, + 2.3181921010000224, + 2.401714166000602, + 2.502357159002713, + 2.5919022890011547, + 2.683453476001887, + 2.78239180500168, + 2.8731444549994194, + 2.9740892090012494, + 3.075472210002772, + 3.1706304460021784, + 3.257111263999832, + 3.348606807001488, + 3.435537874000147, + 3.5295329460022913, + 3.6336748380017525, + 3.7333641910008737, + 3.835431677001907, + 3.938869126002828, + 4.0257487800008676, + 4.114081873001851, + 4.210305013002653, + 4.310247437002545, + 4.413761708001402, + 4.506918737002707, + 4.599990340000659, + 4.681354647000262, + 4.773393389001285, + 4.882587344000058, + 4.985046764999424, + 5.082777759002056, + 5.173370705000707, + 5.273177981001936, + 5.379370418999315, + 5.4883998860022984, + 5.603792918002, + 5.691464632000134, + 5.789049572002114, + 5.893620819002535, + 5.986993200000143, + 6.090064488002099, + 6.190863361000083, + 6.284786468000675, + 6.38988098900154, + 6.490993677001825, + 6.591848980002396, + 6.67772777600112, + 6.76588277799965, + 6.868113802000153, + 6.971234086002369, + 7.066486907999206, + 7.156546376001643, + 7.253140872999211, + 7.348681851999572, + 7.436772989000019, + 7.5330103730011615, + 7.636118387999886, + 7.746103878002032, + 7.870271357001911, + 7.966203046002192, + 8.052038846002688, + 8.135590379002679, + 8.222452165002323, + 8.311465429000236, + 8.393179486000008, + 8.484376864002115, + 8.578409963000013, + 8.658424643999751, + 8.736929624999902, + 8.829461594999884, + 8.921880697002052, + 9.009579806999682, + 9.09071428199968, + 9.180247290001716, + 9.265880609000305, + 9.357097503001569, + 9.456121613002324, + 9.560310105000099, + 9.651787950002472, + 9.747121811000397, + 9.855508765002014, + 9.948456251000607, + 10.032339952002076, + 10.113854693001485, + 10.201176294001925, + 10.296207687999413, + 10.393750426999759, + 10.463941465000971, + 10.536873316999845, + 10.62197266899966, + 10.693461137001577, + 10.762602824001078, + 10.820095058999868, + 10.875143299999763 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 93.37238099760725, + "p95": 109.02946700298344, + "max": 168.8525900026434 + } + }, + { + "start": 30440.521229473, + "first": 30440.689666784, + "last": 30450.378071693, + "end": 30450.378244673, + "elapsed_s": 9.857015199999296, + "chunk_offsets_s": [ + 0.16843731100016157, + 0.42167324500042014, + 0.5282470109996211, + 0.6364688949979609, + 0.7351398799983144, + 0.8309825100004673, + 0.9144905360008124, + 1.0092936859982728, + 1.109864430000016, + 1.1981138589981128, + 1.287130156000785, + 1.3712849660005304, + 1.4594102789997123, + 1.54836932499893, + 1.6402977249999822, + 1.742987057998107, + 1.8473119080008473, + 1.9536085439976887, + 2.046564446998673, + 2.129177738999715, + 2.2265191879996564, + 2.3188126669992926, + 2.402814445998956, + 2.5032685799997125, + 2.5929026550002163, + 2.6844593589994474, + 2.783352783000737, + 2.873489855999651, + 2.9746869720001996, + 3.076025916998333, + 3.1711329789977754, + 3.257605065999087, + 3.3498631010006648, + 3.4367456340005447, + 3.5305215579974174, + 3.634713710998767, + 3.7344087889978255, + 3.835941149998689, + 3.9397547519984073, + 4.026844998999877, + 4.115052974000719, + 4.211169130998314, + 4.311459066997486, + 4.414361980998365, + 4.508178997999494, + 4.601333516999148, + 4.682293301997561, + 4.773978197998076, + 4.883128384000884, + 4.986136076000548, + 5.0837796599989815, + 5.174474343999464, + 5.2743111249983485, + 5.379816661999939, + 5.489609214000666, + 5.605238839998492, + 5.69155180099915, + 5.789548633998493, + 5.893950050998683, + 5.987855414998194, + 6.0909856319995015, + 6.192072335998091, + 6.285943982999015, + 6.391112065000925, + 6.491490276999684, + 6.592408906999481, + 6.678331022998464, + 6.766425240999524, + 6.868590620997566, + 6.972205890997429, + 7.068091402998107, + 7.157890832000703, + 7.25413847299933, + 7.34960395500093, + 7.438130654998531, + 7.5335129060003965, + 7.637230902997544, + 7.747409970997978, + 7.871663355999772, + 7.966713079000328, + 8.052672396999697, + 8.136412534997362, + 8.223360291998688, + 8.312699926998903, + 8.394119868000416, + 8.485082888997567, + 8.579093167998508, + 8.659082439000485, + 8.738134986997466, + 8.830153531998803, + 8.922516087997792, + 9.01046661700093, + 9.092031105999922, + 9.18086835199938, + 9.266873283999303, + 9.35764217399992, + 9.456269883998175, + 9.560900527998456, + 9.652305674997478, + 9.747408121998888, + 9.856842220000544 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 94.92643649900856, + "p95": 109.43409800165682, + "max": 253.23593400025857 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 94.70812367295737, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 10.854811129000154, + "end_to_end_tps": 94.33604950197982, + "timing_valid": true, + "steady_tps": 99.79327790685733, + "per_stream_tps": [ + 24.5, + 25.2, + 23.9, + 26.1 + ], + "ttft_s": [ + 0.32, + 0.24, + 0.17, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.3236061329989752, + "decode_tps": 24.461473535121097, + "prefill_tps": null, + "itl_mean_ms": 40.880611650979574, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.2871709978499, + "round_gap_p50_ms": 94.7414789989125, + "round_gap_p95_ms": 107.70456899990677, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2448580470008892, + "decode_tps": 25.16255779547336, + "prefill_tps": null, + "itl_mean_ms": 39.74158780391935, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.17711600166513, + "round_gap_p50_ms": 94.99035300177638, + "round_gap_p95_ms": 108.00185299740406, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.1659999700023036, + "decode_tps": 23.857667580275017, + "prefill_tps": null, + "itl_mean_ms": 41.915245764710775, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.70672500014189, + "round_gap_p50_ms": 94.84359400084941, + "round_gap_p95_ms": 108.091752998007, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08479472800172516, + "decode_tps": 26.145838378100134, + "prefill_tps": null, + "itl_mean_ms": 38.247004572536646, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.59737600103836, + "round_gap_p50_ms": 95.41305300081149, + "round_gap_p95_ms": 108.30409600021085, + "tokens_per_chunk": 2.5346534653465347 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 40.19611244803659, + "itl_p50_ms": 0.0, + "itl_p95_ms": 104.2871709978499, + "round_gap_p50_ms": 94.97609699974419, + "round_gap_p95_ms": 108.091752998007, + "round_gap_mean_ms": 95.34891789999376, + "tokens_per_chunk": 2.359447004608295, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + } + ], + "client_intervals": [ + { + "start": 30451.399706248, + "first": 30451.723312381, + "last": 30462.147868352, + "end": 30462.147950997, + "elapsed_s": 10.74824474900015, + "chunk_offsets_s": [ + 0.3236061329989752, + 0.407837037997524, + 0.5155416069974308, + 0.6237474489971646, + 0.7216270419994544, + 0.8133789299972705, + 0.8941652559988142, + 0.996507413998188, + 1.1002676419993804, + 1.1878083359988523, + 1.275892344998283, + 1.3575470149989997, + 1.4464339219994145, + 1.5362391409980773, + 1.629798595997272, + 1.7304192349984078, + 1.8299668389990984, + 1.9365941299984115, + 2.0355426719979732, + 2.1192925939976703, + 2.2188669349998236, + 2.313672663000034, + 2.398768729999574, + 2.4980203839986643, + 2.5874229059991194, + 2.6773317549996136, + 2.7759317829986685, + 2.863261889997375, + 2.970353778997378, + 3.074097458000324, + 3.1688389369992365, + 3.25426852199962, + 3.342240834997938, + 3.4303837480001675, + 3.5251757330006512, + 3.629436237999471, + 3.72610000900022, + 3.8240590679997695, + 3.9263854029995855, + 4.019431167998846, + 4.107990519998566, + 4.204100916998868, + 4.303864597997745, + 4.405605951000325, + 4.501043984997523, + 4.59233441899778, + 4.675352533999103, + 4.767286453999986, + 4.871676707000006, + 4.9788364349988115, + 5.079397735000384, + 5.171598843000538, + 5.271350519000407, + 5.375637689998257, + 5.484841630997835, + 5.5999036749999505, + 5.687208038998506, + 5.781349140997918, + 5.880335145000572, + 5.98005578700031, + 6.085699277999083, + 6.18614053499914, + 6.283071934998588, + 6.3862316720005765, + 6.488623303997883, + 6.588933098999405, + 6.675148258000263, + 6.762268463997316, + 6.859370514997863, + 6.962478268000268, + 7.061845580999943, + 7.1518081379981595, + 7.2482418670006155, + 7.343708405998768, + 7.4290751769985945, + 7.526432089998707, + 7.628677084998344, + 7.736785267999949, + 7.8524565329971665, + 7.951449997999589, + 8.041048841998418, + 8.125170855997567, + 8.211110126998392, + 8.299987620997854, + 8.380332315999112, + 8.472835991000466, + 8.56744125700061, + 8.646958557998005, + 8.724265797998669, + 8.814632288998837, + 8.902127864999784, + 8.997103961999528, + 9.07875865599999, + 9.16819667299933, + 9.254770970997924, + 9.346873449998384, + 9.445311996998498, + 9.550517117997515, + 9.641371450998122, + 9.735266289997526, + 9.836957774998154, + 9.929512210997927, + 10.020679323999502, + 10.10278980899966, + 10.187597060998087, + 10.283189060999575, + 10.37931558299897, + 10.44919344200025, + 10.523533452997071, + 10.608164571000088, + 10.680764018998161, + 10.748162103998766 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 94.7414789989125, + "p95": 107.70456899990677, + "max": 115.6712649972178 + } + }, + { + "start": 30451.400103926, + "first": 30451.644961973, + "last": 30461.779066863, + "end": 30461.779154497, + "elapsed_s": 10.379050570998515, + "chunk_offsets_s": [ + 0.2448580470008892, + 0.40719833500043023, + 0.5150004210008774, + 0.6230022739982815, + 0.7211694590005209, + 0.8131664659995295, + 0.893913325999165, + 0.9960594910007785, + 1.0996988539991435, + 1.1874739849990874, + 1.2757446159994288, + 1.3574890790005156, + 1.4460782860005565, + 1.5353891410013603, + 1.6294660679996014, + 1.7301587809997727, + 1.8297075839982426, + 1.936376356999972, + 2.0354580610000994, + 2.118827334998059, + 2.2189064669983054, + 2.3132084120006766, + 2.3979350890003843, + 2.4975553809999838, + 2.586536797000008, + 2.6768869180014008, + 2.7753711990008014, + 2.862786205001612, + 2.970048451999901, + 3.073769630998868, + 3.168384281001636, + 3.2537377779990493, + 3.3417089719987416, + 3.4298342619986215, + 3.525050200998521, + 3.629141641999013, + 3.725422717001493, + 3.8234862339995743, + 3.9260291430000507, + 4.0189594969997415, + 4.1078637879982125, + 4.203175419999752, + 4.303413379999256, + 4.4051559639992774, + 4.500557906001632, + 4.59178270999837, + 4.6748883629988995, + 4.7670761019981, + 4.871218963999127, + 4.978248056999291, + 5.0787594789981085, + 5.17112277400156, + 5.2710146169993095, + 5.375191733000975, + 5.484562626999832, + 5.599731893998978, + 5.686050742999214, + 5.78104109600099, + 5.88019619500119, + 5.979597388999537, + 6.0855365009992965, + 6.186266422999324, + 6.282838202998391, + 6.385528314000112, + 6.488426896998135, + 6.58833031299946, + 6.675026180000714, + 6.7617405019991565, + 6.858886242000153, + 6.961875418001, + 7.061888743999589, + 7.152214759000344, + 7.248175983000692, + 7.342883432000235, + 7.428455886998563, + 7.525617893999879, + 7.628329316998133, + 7.736748120998527, + 7.851985133998824, + 7.951120731999254, + 8.041349455001182, + 8.12482658999943, + 8.211447360998136, + 8.299623221999354, + 8.379654992000724, + 8.472369069000706, + 8.566653384001256, + 8.646395494000899, + 8.723697441000695, + 8.814441653998074, + 8.901635053000064, + 8.996414948000165, + 9.078289032000612, + 9.168155496001418, + 9.254576466999424, + 9.346589057000529, + 9.445351146998291, + 9.550311243998294, + 9.64135147900015, + 9.735653433999687, + 9.836495059000299, + 9.929343789000995, + 10.020068589001312, + 10.102068812000653, + 10.18687607999891, + 10.282850472998689, + 10.378962937000324 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 95.10314600083802, + "p95": 108.00185299740406, + "max": 162.34028799954103 + } + }, + { + "start": 30451.399399033, + "first": 30451.565399003, + "last": 30462.253786673, + "end": 30462.253871621, + "elapsed_s": 10.854472588001954, + "chunk_offsets_s": [ + 0.1659999700023036, + 0.4079650510029751, + 0.51594755500264, + 0.6239936400015722, + 0.7221725480012537, + 0.814067816001625, + 0.8945304890003172, + 0.9966934610019962, + 1.1006473470006313, + 1.188065786001971, + 1.2766917810004088, + 1.3581520740008273, + 1.4464956980009447, + 1.5360334260003583, + 1.6300477770018915, + 1.731036495002627, + 1.8293017650030379, + 1.9373935180010449, + 2.0356551410004613, + 2.1196647510005278, + 2.2189786370036018, + 2.31425271000262, + 2.399156222003512, + 2.498153946002276, + 2.5876860480020696, + 2.6775362880034663, + 2.7758614200029115, + 2.8633265600001323, + 2.970549484001822, + 3.0743312720005633, + 3.1694772580012796, + 3.254373651001515, + 3.342492639003467, + 3.4306335050023335, + 3.5258373380020203, + 3.629793874002644, + 3.7260426150023704, + 3.8245007720033755, + 3.926285456000187, + 4.019246721003583, + 4.108631256000081, + 4.204204767000192, + 4.30378121800095, + 4.406000480001239, + 4.500844074002089, + 4.592359750000469, + 4.675901207003335, + 4.767679177002719, + 4.87261990800107, + 4.978853242002515, + 5.079815996003163, + 5.17220694000207, + 5.271478644001036, + 5.375645126001473, + 5.484962464001001, + 5.600115484001435, + 5.686105673001293, + 5.78169385700312, + 5.8809690850030165, + 5.980247303003125, + 6.085923049002304, + 6.186390788003337, + 6.283422823002184, + 6.386328389002301, + 6.489530941002158, + 6.589350352001929, + 6.675500858000305, + 6.7626374540013785, + 6.85973595500036, + 6.962415853002312, + 7.062210846001108, + 7.152302311002131, + 7.248611641000025, + 7.343960594000237, + 7.429108199001348, + 7.5260799860006955, + 7.628926234003302, + 7.737404336003237, + 7.8528776570019545, + 7.951540206002392, + 8.0413142230027, + 8.126267834002647, + 8.211559604002105, + 8.300087185001757, + 8.38030199500281, + 8.472929094001302, + 8.567579153001134, + 8.64734248500099, + 8.7246489570025, + 8.814985176002665, + 8.902203921003093, + 8.996928789001686, + 9.079350330001034, + 9.168567662003625, + 9.255135452000104, + 9.346839627000008, + 9.445907286000875, + 9.550614011001016, + 9.641446259000077, + 9.736227163000876, + 9.837139728002512, + 9.929971171000943, + 10.020722932000353, + 10.10238814600234, + 10.187677019002876, + 10.283217814001546, + 10.38035769400085, + 10.44941022400235, + 10.523961277001945, + 10.608355495001888, + 10.68052758200065, + 10.748167669000395, + 10.801808249001624, + 10.854387640003551 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 94.84359400084941, + "p95": 108.091752998007, + "max": 241.9650810006715 + } + }, + { + "start": 30451.399060492, + "first": 30451.48385522, + "last": 30461.236841386, + "end": 30461.237047295, + "elapsed_s": 9.837986802998785, + "chunk_offsets_s": [ + 0.08479472800172516, + 0.4081053289992269, + 0.5161272190016462, + 0.6246817729988834, + 0.7223344939993694, + 0.8142849010000646, + 0.8947356120006589, + 0.9969801090010151, + 1.1008232050007791, + 1.1885691520001274, + 1.2770848539985309, + 1.3584464140003547, + 1.4468854119986645, + 1.5366069150004478, + 1.630573276001087, + 1.7309876719991735, + 1.8302917549990525, + 1.938005546999193, + 2.036349447000248, + 2.1202500740000687, + 2.2203789400009555, + 2.314548936999927, + 2.3997167749985238, + 2.498706343001686, + 2.587844508001581, + 2.6776318039992475, + 2.7768178769983933, + 2.864151966001373, + 2.971346951999294, + 3.0745697370002745, + 3.1697535599996627, + 3.254653422998672, + 3.342709148000722, + 3.4312486369999533, + 3.52604593200158, + 3.6300297009984206, + 3.726918314001523, + 3.824449948999245, + 3.9269791539991274, + 4.019866891001584, + 4.108727109000029, + 4.204310070999782, + 4.304333741998562, + 4.406505573999311, + 4.501489941998443, + 4.592878883999219, + 4.6765073509996, + 4.768075623000186, + 4.8726729990012245, + 4.979385424998327, + 5.079866336000123, + 5.172301945000072, + 5.271916014000453, + 5.376309015999141, + 5.4854486389995145, + 5.600505213998986, + 5.687023414000578, + 5.781919081000524, + 5.881052048000129, + 5.980508028998884, + 6.0867193340018275, + 6.187044859998423, + 6.28423505200044, + 6.386624088001554, + 6.489672176998283, + 6.589655919000506, + 6.675592023999343, + 6.76283403799971, + 6.860493109001254, + 6.9626920750015415, + 7.062643576999108, + 7.152993307998258, + 7.248834018999332, + 7.344626147001691, + 7.429818770000566, + 7.526830550999875, + 7.6290574119993835, + 7.737361507999594, + 7.853275767000014, + 7.952022672001476, + 8.041892878001818, + 8.12657189699894, + 8.212196356998902, + 8.300699917999736, + 8.380580664001172, + 8.473228407001443, + 8.567957384999318, + 8.647641542000201, + 8.724954187000549, + 8.815404649998527, + 8.902604381000856, + 8.998017434001667, + 9.079642063999927, + 9.168974103999062, + 9.255701474001398, + 9.347563982999418, + 9.446325065000565, + 9.55120837099821, + 9.642058369001461, + 9.73650325700146, + 9.83778089399857 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 95.49800750028226, + "p95": 108.30409600021085, + "max": 323.31060099750175 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 1.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 103.3, + "per_stream_tps": 22.0, + "ttft_s_max": 0.43, + "request": { + "ttft_s": 0.253, + "decode_tps": 22.018, + "prefill_tps": null, + "itl_mean_ms": 45.417, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.792, + "round_gap_p50_ms": 105.162, + "round_gap_p95_ms": 120.28, + "tokens_per_chunk": 2.286 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 45.389, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.622, + "round_gap_p50_ms": 105.311, + "round_gap_p95_ms": 120.292, + "tokens_per_chunk": 2.315 + }, + "timing_valid": true, + "end_to_end_tps": 103.0110518435381, + "makespan_s": 12.425851178999437, + "steady_tps": 110.2, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 5, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 103.48777324491012, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 12.409923448998597, + "end_to_end_tps": 103.14326315230316, + "timing_valid": true, + "steady_tps": 109.94886612062979, + "per_stream_tps": [ + 22.1, + 22.2, + 21.6, + 23.9, + 20.9 + ], + "ttft_s": [ + 0.43, + 0.09, + 0.35, + 0.27, + 0.18 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.4279248409984575, + "decode_tps": 22.060174178837702, + "prefill_tps": null, + "itl_mean_ms": 45.33055776863715, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.51580000013928, + "round_gap_p50_ms": 104.87707200081786, + "round_gap_p95_ms": 120.09087600017665, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08940868700301507, + "decode_tps": 22.24584565471295, + "prefill_tps": null, + "itl_mean_ms": 44.95221334901883, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.69565900208545, + "round_gap_p50_ms": 105.16244300015387, + "round_gap_p95_ms": 120.70671000037692, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3483882319997065, + "decode_tps": 21.637815969459822, + "prefill_tps": null, + "itl_mean_ms": 46.21538520391457, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.62245900131529, + "round_gap_p50_ms": 105.01914300039061, + "round_gap_p95_ms": 120.4099739989033, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.26587719099916285, + "decode_tps": 23.874675840427734, + "prefill_tps": null, + "itl_mean_ms": 41.885385447063065, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.91478699847357, + "round_gap_p50_ms": 105.46347200215678, + "round_gap_p95_ms": 120.19036999845412, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.18156625800111215, + "decode_tps": 20.853360026301097, + "prefill_tps": null, + "itl_mean_ms": 47.95390281176557, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.61426100161043, + "round_gap_p50_ms": 104.31216100187157, + "round_gap_p95_ms": 120.2802280022297, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 45.267488916079834, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.62245900131529, + "round_gap_p50_ms": 105.01914300039061, + "round_gap_p95_ms": 120.2920930008986, + "round_gap_mean_ms": 105.32125614598867, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 30462.258498757, + "first": 30462.686423598, + "last": 30474.245715829, + "end": 30474.245884909, + "elapsed_s": 11.98738615199909, + "chunk_offsets_s": [ + 0.4279248409984575, + 0.5188747389984201, + 0.6303905409986328, + 0.7462929520006583, + 0.8537822340003913, + 0.9532843560009496, + 1.0439896610005235, + 1.1561056479986291, + 1.2638476779975463, + 1.358500267000636, + 1.457772030000342, + 1.549029021000024, + 1.644419496999035, + 1.7396865659975447, + 1.850709320999158, + 1.9624923199990008, + 2.078735520997725, + 2.200259263998305, + 2.304083453000203, + 2.3926703100005398, + 2.5027614229984465, + 2.605846840000595, + 2.6979905599982885, + 2.807947236000473, + 2.912457213999005, + 3.0106880659986928, + 3.12631541799783, + 3.2248382449979545, + 3.3326043829983973, + 3.437137613000232, + 3.5373800589986786, + 3.6393805789994076, + 3.738295379000192, + 3.832702589999826, + 3.9329676300003484, + 4.048378524999862, + 4.159221974998218, + 4.262549068000226, + 4.369643829999404, + 4.461116266000317, + 4.559465546000865, + 4.66493832299966, + 4.773243913998158, + 4.880601069999102, + 4.98631877599837, + 5.0920479640008125, + 5.186175015998742, + 5.28755609899963, + 5.399648774000525, + 5.514615778000007, + 5.622093140998913, + 5.7220621749984275, + 5.831794801997603, + 5.950600628999382, + 6.070892722000281, + 6.197024369997962, + 6.293465831000503, + 6.400218055998266, + 6.504272309000953, + 6.603894698997465, + 6.7149173129982955, + 6.835008188998472, + 6.941677865997917, + 7.0511889469999005, + 7.170029704000626, + 7.2808608659979654, + 7.377023141998507, + 7.482732086999022, + 7.591737626000395, + 7.704335375998198, + 7.809760533000372, + 7.914325224999629, + 8.027654563000397, + 8.13440151299801, + 8.233247159998427, + 8.341309442999773, + 8.452524390999315, + 8.574133760997938, + 8.709408935999818, + 8.81145389700032, + 8.90388856699792, + 8.994817203998537, + 9.103772095997556, + 9.22280412300097, + 9.317521672001021, + 9.414285885999561, + 9.519630077000329, + 9.609889805000421, + 9.696472501000244, + 9.79176513999846, + 9.896642211999279, + 10.007082493997586, + 10.106925395000872, + 10.21715567299907, + 10.315052911999373, + 10.42155501399975, + 10.526856489999773, + 10.64085183899806, + 10.738144045000809, + 10.839272564000566, + 10.946586340000067, + 11.04307707299813, + 11.144614581000496, + 11.234555734998139, + 11.332162465998408, + 11.43532987999788, + 11.55084567999802, + 11.641015733999666, + 11.730863378998038, + 11.825280478999048, + 11.904715474000113, + 11.98721707200093 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 104.87707200081786, + "p95": 120.09087600017665, + "max": 135.2751750018797 + } + }, + { + "start": 30462.257616785, + "first": 30462.347025472, + "last": 30473.809839876, + "end": 30473.810036078, + "elapsed_s": 11.552419293002458, + "chunk_offsets_s": [ + 0.08940868700301507, + 0.5188558860027115, + 0.6324222650000593, + 0.7470924389999709, + 0.8539041390031343, + 0.9533770920024836, + 1.0450861450008233, + 1.1566495890001534, + 1.2640051010021125, + 1.3591979830016498, + 1.4583139230016968, + 1.549467545002699, + 1.6448000990021683, + 1.7411365450025187, + 1.850768042000709, + 1.962692312001309, + 2.079468291001831, + 2.2004174690009677, + 2.3044736020019627, + 2.394216287000745, + 2.50377163200028, + 2.6065600520014414, + 2.6985648040026717, + 2.808912076001434, + 2.913229436002439, + 3.0118078500017873, + 3.1269825260023936, + 3.2253356979999808, + 3.332674170000246, + 3.437610359000246, + 3.5383730940011446, + 3.6403703030009638, + 3.7386029780027457, + 3.8334957440019934, + 3.933591912002157, + 4.049506576000567, + 4.160369680001168, + 4.263107624003169, + 4.370431819002988, + 4.46176661500067, + 4.560251920000155, + 4.665414363000309, + 4.77333799200278, + 4.880784126002254, + 4.987089031001233, + 5.092674629002431, + 5.18640483100171, + 5.289276882002014, + 5.400772908000363, + 5.515099573000043, + 5.622270792002382, + 5.723006595002516, + 5.832405779001419, + 5.950939907001157, + 6.071646617001534, + 6.198100865000015, + 6.295188310003141, + 6.400916140002664, + 6.505237166002189, + 6.60333151800296, + 6.715476829002, + 6.835313677001977, + 6.942252446002385, + 7.05195513900253, + 7.170972061001521, + 7.281268333001208, + 7.377443626002787, + 7.4838306510027905, + 7.5919387840003765, + 7.705633468001906, + 7.810069874001783, + 7.915020782002102, + 8.028871753002022, + 8.13486204000219, + 8.234543029000633, + 8.342335212000762, + 8.452850364003098, + 8.574327002002974, + 8.710035777003213, + 8.811715823001578, + 8.904313161001483, + 8.995314160001726, + 9.104001527000946, + 9.223090996001702, + 9.31763237999985, + 9.415659779002453, + 9.520289988002332, + 9.610318988001382, + 9.697702852001385, + 9.792603342000803, + 9.897828698001831, + 10.007742789002805, + 10.107250376000593, + 10.217926551002165, + 10.316375823000271, + 10.422737262000737, + 10.527517664002517, + 10.642412819001038, + 10.738942988002236, + 10.839813929000229, + 10.9475328059998, + 11.043877168001927, + 11.145222359002219, + 11.235381193000649, + 11.333381543001451, + 11.43652743200073, + 11.552223091002816 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 105.19389950059121, + "p95": 120.70671000037692, + "max": 429.4471989996964 + } + }, + { + "start": 30462.257688204, + "first": 30462.606076436, + "last": 30474.390999663, + "end": 30474.39111298, + "elapsed_s": 12.13342477599872, + "chunk_offsets_s": [ + 0.3483882319997065, + 0.5194405879992701, + 0.6328322429981199, + 0.7472643490000337, + 0.8543362489981519, + 0.9535871899970516, + 1.0449380459976965, + 1.1564828279988433, + 1.2644042520005314, + 1.3598325959974318, + 1.458748511999147, + 1.5495530839980347, + 1.6448130360004143, + 1.7412740889994893, + 1.8516783669983852, + 1.9627513209998142, + 2.079475917998934, + 2.2017686359977233, + 2.3042090839990124, + 2.3939149719990382, + 2.5035360589972697, + 2.6071464819979155, + 2.697933404997457, + 2.808404805000464, + 2.912960806999763, + 3.0113694539977587, + 3.126991912999074, + 3.2255094469983305, + 3.332876160999149, + 3.4380260619982437, + 3.5386522919980052, + 3.6400671169976704, + 3.738620855998306, + 3.8337417119983, + 3.9336500729987165, + 4.049100243999419, + 4.159289314997295, + 4.263656409999385, + 4.370503062000353, + 4.462169124999491, + 4.560942855998292, + 4.665686508997169, + 4.773516154997196, + 4.880981095997413, + 4.986793648000457, + 5.092819817000418, + 5.187222852997365, + 5.288682870999764, + 5.399979177000205, + 5.515721584997664, + 5.622617826997157, + 5.723337286999595, + 5.832902480997291, + 5.951456021997728, + 6.0718209099977685, + 6.1974581590002344, + 6.294363889999659, + 6.4011318199991365, + 6.504847943997447, + 6.603611452999758, + 6.715178184000251, + 6.835588157999155, + 6.942984864999744, + 7.052111105997028, + 7.170677013997192, + 7.281619381999917, + 7.377564474998508, + 7.482986290000554, + 7.592722583998693, + 7.705188002997602, + 7.8109143000001495, + 7.915367321998929, + 8.02903988899925, + 8.13492044099985, + 8.234637795998424, + 8.342522058999748, + 8.452891478998936, + 8.574831652000285, + 8.710173018000205, + 8.812393566997343, + 8.905380781998247, + 8.99574439999924, + 9.104355070998281, + 9.223157344997162, + 9.31781737199708, + 9.414859601998614, + 9.520342920997791, + 9.611003688998608, + 9.697236918000272, + 9.792322815999796, + 9.897341959000187, + 10.007860950998293, + 10.107585562000168, + 10.218070076000004, + 10.315801656997792, + 10.422119661998295, + 10.527756292998674, + 10.641587886999332, + 10.738580903998809, + 10.84018058699803, + 10.947689396998612, + 11.044140022997453, + 11.145583099998476, + 11.235406970998156, + 11.333078691997798, + 11.43620966099843, + 11.551737821999268, + 11.641622183997242, + 11.731735659999686, + 11.825858656000491, + 11.905587179997383, + 11.987821187998634, + 12.059716427997046, + 12.133311458997923 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 105.01914300039061, + "p95": 120.4099739989033, + "max": 171.0523559995636 + } + }, + { + "start": 30462.25797228, + "first": 30462.523849471, + "last": 30473.20462276, + "end": 30473.204770076, + "elapsed_s": 10.946797796001192, + "chunk_offsets_s": [ + 0.26587719099916285, + 0.5201373599993531, + 0.631366303001414, + 0.7470943100015575, + 0.8538938400015468, + 0.953905615999247, + 1.0447874199999205, + 1.1565766019994044, + 1.2638468469995132, + 1.3593051110001397, + 1.4586563039993052, + 1.549357505999069, + 1.6448549179986003, + 1.740156735999335, + 1.8506624170004216, + 1.962837483999465, + 2.0790031409997027, + 2.201603729998169, + 2.304873280998436, + 2.3939553189993603, + 2.5030686700010847, + 2.6064396020010463, + 2.6977842189990042, + 2.8082636459985224, + 2.9127402500016615, + 3.0111444829999527, + 3.1270592699984263, + 3.2246328799992625, + 3.3327777160011465, + 3.43716991600013, + 3.538595936999627, + 3.64042617800078, + 3.73863366599835, + 3.833735139000055, + 3.932942073999584, + 4.049384800000553, + 4.15919421199942, + 4.263559772000008, + 4.37042284799827, + 4.46210976400107, + 4.56008824500168, + 4.665268422999361, + 4.773499764000007, + 4.881190298998263, + 4.986314904999745, + 5.092806960001326, + 5.18639416500082, + 5.287993630001438, + 5.399517609999748, + 5.514942917998269, + 5.6224652669989155, + 5.72249583699886, + 5.832871424998302, + 5.950852145000681, + 6.071042514999135, + 6.19746938499884, + 6.294050166001398, + 6.400790908999625, + 6.504437454001163, + 6.602859779999562, + 6.715053577001527, + 6.8352291289993445, + 6.9423188119981205, + 7.0519457519985735, + 7.170293885999854, + 7.2812721070004045, + 7.376996450999286, + 7.482889075999992, + 7.592171110998606, + 7.705141083999479, + 7.810433557999204, + 7.914755720001267, + 8.028794465000828, + 8.135012890001235, + 8.233702826000808, + 8.342876386999706, + 8.452811712999392, + 8.574130017001153, + 8.709854398999596, + 8.811790439998731, + 8.904357618001086, + 8.995246163998672, + 9.104146604000562, + 9.22309625699927, + 9.317769412999041, + 9.41452125000069, + 9.519824549999612, + 9.610369139001705, + 9.696825451999757, + 9.791893215999153, + 9.897119195000414, + 10.007468404000974, + 10.107386193998536, + 10.217723536999983, + 10.315653095000016, + 10.421974553999462, + 10.527438026001619, + 10.64156511499823, + 10.738358523998613, + 10.839846169001248, + 10.946650480000244 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 105.67804850143148, + "p95": 120.19036999845412, + "max": 254.26016900019022 + } + }, + { + "start": 30462.257510009, + "first": 30462.439076267, + "last": 30474.667321484, + "end": 30474.667433458, + "elapsed_s": 12.409923448998597, + "chunk_offsets_s": [ + 0.18156625800111215, + 0.5202582089987118, + 0.6312724330018682, + 0.7468649100010225, + 0.8539259989993297, + 0.9539687670003332, + 1.045055537000735, + 1.1565565169985348, + 1.2641722609987482, + 1.359613142998569, + 1.4586636120002368, + 1.5496559899984277, + 1.645112287998927, + 1.741303369999514, + 1.8513750370002526, + 1.9632363800010353, + 2.079829062000499, + 2.200797399000294, + 2.3048885530006373, + 2.393465590001142, + 2.50381407399982, + 2.60711552200155, + 2.6985654429990973, + 2.8090818599994236, + 2.913394021001295, + 3.0120993290001934, + 3.1270219450016157, + 3.2253666409997095, + 3.3341593320001266, + 3.437955298999441, + 3.538635900000372, + 3.64076830400154, + 3.7384118449990638, + 3.8333683380005823, + 3.933311050001066, + 4.049699675000738, + 4.1599286460004805, + 4.263340590001462, + 4.3703284969997185, + 4.461549270999967, + 4.560798993999924, + 4.665203960001236, + 4.773765640999045, + 4.881012598998495, + 4.987245604999771, + 5.093106180000177, + 5.186346621001576, + 5.288945614000113, + 5.400549247999152, + 5.5153273420000914, + 5.6222746610001195, + 5.7233006979986385, + 5.8324416799987375, + 5.951099934998638, + 6.0713801630008675, + 6.197424895999575, + 6.294982655999775, + 6.400943676999304, + 6.5053977700008545, + 6.603475347001222, + 6.715623025000241, + 6.835324439001852, + 6.942256220001582, + 7.052111297001829, + 7.170960930001456, + 7.281449789999897, + 7.377285356000357, + 7.482942279999406, + 7.593117819000327, + 7.705813405998924, + 7.81023789900064, + 7.914969624001969, + 8.028433356001187, + 8.135284092000802, + 8.234738670998922, + 8.34280656700139, + 8.452659695001785, + 8.574382236998645, + 8.710071773999516, + 8.811747805000778, + 8.904468423999788, + 8.995291866998741, + 9.103993450000416, + 9.223391670999263, + 9.317626287000166, + 9.415522728999349, + 9.520444275000045, + 9.610530366000603, + 9.697875082001701, + 9.792146170999331, + 9.897748227998818, + 10.008000493999134, + 10.107619496000552, + 10.217950474001555, + 10.316705715998978, + 10.422905814000842, + 10.527683354001056, + 10.642157845999463, + 10.73919209000087, + 10.839838251999026, + 10.947820960001991, + 11.044269571000768, + 11.145713464000437, + 11.235390306999761, + 11.332754139999452, + 11.436516716999904, + 11.552130978001514, + 11.641700030999345, + 11.731486188000417, + 11.825733312998636, + 11.905255574998591, + 11.987848197000858, + 12.060185528000147, + 12.134092189000512, + 12.185104015999968, + 12.241068602001178, + 12.303782282000611, + 12.352189107001323, + 12.409811475001334 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 104.35856350159156, + "p95": 120.2802280022297, + "max": 338.69195099759963 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 103.31286632443326, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 12.425851178999437, + "end_to_end_tps": 103.0110518435381, + "timing_valid": true, + "steady_tps": 110.21388875374262, + "per_stream_tps": [ + 21.4, + 22.4, + 21.4, + 24.2, + 21.1 + ], + "ttft_s": [ + 0.08, + 0.17, + 0.25, + 0.42, + 0.34 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08381232600004296, + "decode_tps": 21.377448455211766, + "prefill_tps": null, + "itl_mean_ms": 46.77826739215936, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.28868800107739, + "round_gap_p50_ms": 105.39784399952623, + "round_gap_p95_ms": 120.99283199859201, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16662630399878253, + "decode_tps": 22.36410324660584, + "prefill_tps": null, + "itl_mean_ms": 44.71451365490223, + "itl_p50_ms": 0.0, + "itl_p95_ms": 116.04578900005436, + "round_gap_p50_ms": 105.48527100036154, + "round_gap_p95_ms": 121.50500000279862, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25339806500051054, + "decode_tps": 21.433417183782797, + "prefill_tps": null, + "itl_mean_ms": 46.656116074511516, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.79156900188536, + "round_gap_p50_ms": 105.22134699931485, + "round_gap_p95_ms": 120.89047799963737, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.4238658670001314, + "decode_tps": 24.157975271137634, + "prefill_tps": null, + "itl_mean_ms": 41.394197517651016, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.81386499892687, + "round_gap_p50_ms": 105.40205299912486, + "round_gap_p95_ms": 120.76291700213915, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33882880799865234, + "decode_tps": 21.097189135020635, + "prefill_tps": null, + "itl_mean_ms": 47.39967934117029, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.91599999883329, + "round_gap_p50_ms": 103.94547800024156, + "round_gap_p95_ms": 121.51429100049427, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 45.38855479607888, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.81386499892687, + "round_gap_p50_ms": 105.31050899953698, + "round_gap_p95_ms": 121.07072000071639, + "round_gap_mean_ms": 105.60293314781127, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 30474.67306078, + "first": 30474.756873106, + "last": 30486.685331291, + "end": 30486.685703148, + "elapsed_s": 12.012642368001252, + "chunk_offsets_s": [ + 0.08381232600004296, + 0.5134294610033976, + 0.6226794630019867, + 0.7464139750009053, + 0.8539382630006003, + 0.9530260420033301, + 1.0446849340005429, + 1.1488800770021044, + 1.2559160070013604, + 1.351799074000155, + 1.4504950830014423, + 1.5430597720005608, + 1.639976367001509, + 1.7424031170012313, + 1.8549615630035987, + 1.9660626850018161, + 2.0809687960027077, + 2.197516064003139, + 2.3003252840026107, + 2.3911357220022182, + 2.5017589290000615, + 2.6038607870032138, + 2.7008984660024, + 2.8141068480035756, + 2.918396085002314, + 3.0171165250030754, + 3.126687785003014, + 3.223069909003243, + 3.3312545840017265, + 3.4364770790016337, + 3.540383344003203, + 3.6414264970007935, + 3.7468243410003197, + 3.8407610270005534, + 3.941256208003324, + 4.0566392070031725, + 4.159345192001638, + 4.263390452000749, + 4.369034229002864, + 4.4599658050028665, + 4.559965397002088, + 4.665894035002566, + 4.78028015700329, + 4.889707674003148, + 4.996631189002073, + 5.0995448420035245, + 5.190623947000859, + 5.293255773001874, + 5.404502666002372, + 5.52079135400345, + 5.62858547200085, + 5.735673539002164, + 5.847232799002086, + 5.964755186003458, + 6.083834655000828, + 6.2048142710009415, + 6.303323820000514, + 6.410429538002063, + 6.516336130000127, + 6.616133319002984, + 6.734857227002067, + 6.855850059000659, + 6.9615851080016, + 7.072314170000027, + 7.183396779000759, + 7.29382079500283, + 7.389681049000501, + 7.495520999000291, + 7.608281088003423, + 7.730105746002664, + 7.837508761000208, + 7.9462768790035625, + 8.057708710002771, + 8.158400306001568, + 8.256665119002719, + 8.36621438200018, + 8.477388326002256, + 8.599310627003433, + 8.741651369000465, + 8.844618684001034, + 8.937622516001284, + 9.026680096001655, + 9.130504077002115, + 9.248939216002327, + 9.342933424002695, + 9.440267718000541, + 9.548043810002127, + 9.636764754002797, + 9.72960538100233, + 9.825438561001647, + 9.932138728003338, + 10.040226915003586, + 10.134076403002837, + 10.24271879400112, + 10.339118345000315, + 10.446016372003214, + 10.551458542002365, + 10.66604714800269, + 10.770378482000524, + 10.870545754001796, + 10.978443474003143, + 11.076153357000294, + 11.171930587002862, + 11.260912192003161, + 11.359306306003418, + 11.463143104003393, + 11.569028814003104, + 11.658847523001896, + 11.754368142002932, + 11.850565357002779, + 11.930686679002974, + 12.012270511000679 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 105.39784399952623, + "p95": 120.99283199859201, + "max": 429.61713500335463 + } + }, + { + "start": 30474.673159785, + "first": 30474.839786089, + "last": 30486.241987071, + "end": 30486.242239085, + "elapsed_s": 11.56907929999943, + "chunk_offsets_s": [ + 0.16662630399878253, + 0.5130724619993998, + 0.6228874349981197, + 0.7463804399994842, + 0.8536806679985602, + 0.953217095000582, + 1.044438500997785, + 1.1484720799999195, + 1.2558855429997493, + 1.3515669699991122, + 1.450469000999874, + 1.5431262489983055, + 1.6403660689975368, + 1.7423954400001094, + 1.8551906760003476, + 1.9659068449982442, + 2.0807046919981076, + 2.1976354260004882, + 2.300151821000327, + 2.391392404999351, + 2.502054311997199, + 2.6038290269971185, + 2.7010761979981908, + 2.8143908439997176, + 2.9177640919988335, + 3.0173520669995924, + 3.126663140999881, + 3.223261137998634, + 3.3311915769991174, + 3.436676847999479, + 3.5406951339973602, + 3.6413787130004494, + 3.7466086610002094, + 3.8407633289971272, + 3.9413474559987662, + 4.056610724997881, + 4.159137906997785, + 4.263870554997993, + 4.369308743000147, + 4.460168739999062, + 4.560157760999573, + 4.665836816999217, + 4.780056383999181, + 4.889905058997101, + 4.9965811829970335, + 5.099619107997569, + 5.190854019998369, + 5.293108456997288, + 5.40476878499976, + 5.5208145739998145, + 5.628384919997188, + 5.735613505999936, + 5.847430792997329, + 5.964610187998915, + 6.084077441999398, + 6.205032989997562, + 6.30341577299987, + 6.410269059997518, + 6.516281677999359, + 6.616080050000164, + 6.734184565997566, + 6.855689566000365, + 6.961440078997839, + 7.072281706998183, + 7.182601205997344, + 7.293798134000099, + 7.390209223998681, + 7.495745522999641, + 7.608428163999633, + 7.730064294999465, + 7.837729078997654, + 7.94601434300057, + 8.057526691998646, + 8.158012058000168, + 8.256803975000366, + 8.366057725997962, + 8.47707971699856, + 8.599258573998668, + 8.740839308000432, + 8.844795663997502, + 8.937595041999884, + 9.026616384999215, + 9.130708532000426, + 9.248936001000402, + 9.342965919000562, + 9.440485029997944, + 9.547719000998768, + 9.636346010000125, + 9.729032795999956, + 9.82574533099978, + 9.932104952997179, + 10.040398090997769, + 10.134159671997622, + 10.24323841399746, + 10.33960553399811, + 10.446512867998536, + 10.55175330199927, + 10.665809125999658, + 10.77035305499885, + 10.870633148999332, + 10.978691681997589, + 11.076217036999878, + 11.171900026998628, + 11.260907489999227, + 11.359566155999346, + 11.463248233998456, + 11.56882728599885 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 105.51078500066069, + "p95": 121.50500000279862, + "max": 346.4461580006173 + } + }, + { + "start": 30474.672955331, + "first": 30474.926353396, + "last": 30486.823662995, + "end": 30486.823787011, + "elapsed_s": 12.1508316799991, + "chunk_offsets_s": [ + 0.25339806500051054, + 0.5137783520003723, + 0.6236775210018095, + 0.7463136750011472, + 0.8541215440018277, + 0.953040129999863, + 1.045218865001516, + 1.149315883001691, + 1.2561672529991483, + 1.3524673669999174, + 1.450862956000492, + 1.543628773000819, + 1.6404795620001096, + 1.743118090002099, + 1.8552521640012856, + 1.966544563001662, + 2.0814676090012654, + 2.1985943210020196, + 2.300530409000203, + 2.3922927870007697, + 2.5025097239995375, + 2.6038221189992328, + 2.701424881001003, + 2.814388524999231, + 2.9181813479990524, + 3.018219585999759, + 3.1273518890011474, + 3.223977775000094, + 3.331176128001971, + 3.437638750001497, + 3.541011706001882, + 3.6414378180015774, + 3.746979604999069, + 3.8410216599986597, + 3.941832101001637, + 4.057175584999641, + 4.159153945998696, + 4.2639569739985745, + 4.369761116999143, + 4.46083583500149, + 4.560998094999377, + 4.666308603998914, + 4.780705551998835, + 4.890603722000378, + 4.997231630000897, + 5.100079222000204, + 5.191272059000767, + 5.293676499000867, + 5.405538401999365, + 5.521329971001251, + 5.629060073999426, + 5.735865262999141, + 5.848009692999767, + 5.964706538001337, + 6.084956507002062, + 6.205846985001699, + 6.304086386000563, + 6.410094382001262, + 6.516167896999832, + 6.616638209001394, + 6.734842210000352, + 6.856226568001148, + 6.961447915000463, + 7.072933338000439, + 7.183183977002045, + 7.293784845001937, + 7.390663693000533, + 7.49637380599961, + 7.609190202001628, + 7.72978277600123, + 7.838448867001716, + 7.946590572999412, + 8.057609032999608, + 8.158285260000412, + 8.256922153999767, + 8.36641902699921, + 8.477128523001738, + 8.599695132001216, + 8.74120160000166, + 8.84503689900157, + 8.938554434000253, + 9.026738769000076, + 9.1311196630013, + 9.24966986200161, + 9.343404524999642, + 9.441027773002133, + 9.547700162002002, + 9.637005765001959, + 9.729486642001575, + 9.826640260998829, + 9.932707570002094, + 10.040391804999672, + 10.134670079998614, + 10.24381745899882, + 10.340024357999937, + 10.446387907999451, + 10.55256441900201, + 10.666550983001798, + 10.770626097000786, + 10.871557295999082, + 10.979201546000695, + 11.076190824998775, + 11.17186226999911, + 11.26126361000206, + 11.360065786000632, + 11.463705533999018, + 11.5698264820021, + 11.658632417002082, + 11.754854834998696, + 11.851137173001916, + 11.931040544001007, + 12.013347938998777, + 12.085283032000007, + 12.150707664000947 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 105.22134699931485, + "p95": 120.89047799963737, + "max": 260.3802869998617 + } + }, + { + "start": 30474.672718158, + "first": 30475.096584025, + "last": 30485.652104392, + "end": 30485.652281565, + "elapsed_s": 10.97956340699966, + "chunk_offsets_s": [ + 0.4238658670001314, + 0.5141107560011733, + 0.6237075850003748, + 0.7468891680000525, + 0.8538929580026888, + 0.9532264020017465, + 1.0452592760011612, + 1.1494666050020896, + 1.2568700890005857, + 1.3524737799998547, + 1.4513421320007183, + 1.5436589810014993, + 1.6409529170014139, + 1.7433118289991398, + 1.8551773069993942, + 1.966641425002308, + 2.081959226001345, + 2.1984129270022095, + 2.3008926210022764, + 2.3922759630004293, + 2.502971341000375, + 2.6046057130006375, + 2.7010163410013774, + 2.814231022999593, + 2.9185096100009105, + 3.0181600809992233, + 3.127751489999355, + 3.2240344660021947, + 3.3313337750005303, + 3.437611355999252, + 3.541556896001566, + 3.641897213001357, + 3.747260482999991, + 3.8414000709999527, + 3.9417361820014776, + 4.0575500470004044, + 4.15932558500208, + 4.263610960999358, + 4.3701386009997805, + 4.46114607400159, + 4.561128203000408, + 4.6663766330020735, + 4.780348312000569, + 4.890916472002573, + 4.997388333002164, + 5.100372781002079, + 5.191645898001298, + 5.294157274001918, + 5.4055614300013985, + 5.521766801000922, + 5.629117963999306, + 5.735938600999361, + 5.848196172002645, + 5.964981100001751, + 6.084775287999946, + 6.20589091399961, + 6.305225045001862, + 6.410627098000987, + 6.5164524690007966, + 6.616922061999503, + 6.734833926002466, + 6.856521023000823, + 6.961803665999469, + 7.072985646002053, + 7.1839963250022265, + 7.294278317000135, + 7.390967919000104, + 7.496563532000437, + 7.609490894999908, + 7.730253812002047, + 7.83892935500262, + 7.947019183000521, + 8.05774589000066, + 8.158336002001306, + 8.257106138000381, + 8.366416516000754, + 8.477164759999141, + 8.599986549001187, + 8.741626676001033, + 8.845037858001888, + 8.938541864001309, + 9.026927598999464, + 9.130786585999886, + 9.249626524000632, + 9.343355829001666, + 9.441326978001598, + 9.548479920002137, + 9.63732918200185, + 9.729621180002141, + 9.826581698001974, + 9.93274921199918, + 10.040449631000229, + 10.135032213001978, + 10.244006097000238, + 10.340607768001064, + 10.447231935999298, + 10.552556118000211, + 10.665890011001466, + 10.770278308002162, + 10.8717109600002, + 10.97938623400114 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 105.49883299972862, + "p95": 120.76291700213915, + "max": 141.64012699984596 + } + }, + { + "start": 30474.672280129, + "first": 30475.011108937, + "last": 30487.098027169, + "end": 30487.098131308, + "elapsed_s": 12.425851178999437, + "chunk_offsets_s": [ + 0.33882880799865234, + 0.5148128079999879, + 0.6242725729971426, + 0.7471119969995925, + 0.85425887999736, + 0.9546433949981292, + 1.0462365779967513, + 1.1500322789979691, + 1.2572269119991688, + 1.3526722209971922, + 1.4514454700001806, + 1.5441615049967368, + 1.64052800099671, + 1.7436203099969134, + 1.8555535599989526, + 1.9671779309974227, + 2.0825548859975243, + 2.1989684629988915, + 2.300937541996973, + 2.392116045997682, + 2.5028749269986292, + 2.6052312609972432, + 2.701362000996596, + 2.81521534499916, + 2.918731862999266, + 3.01806767999733, + 3.1277097829988634, + 3.2240712940001686, + 3.331963463999273, + 3.437804758999846, + 3.5410487099979946, + 3.6424517249979544, + 3.7475567609981226, + 3.8417732219968457, + 3.9421035119994485, + 4.057576655999583, + 4.159965509999893, + 4.264507472998957, + 4.370100395997724, + 4.461117959999683, + 4.560994751998805, + 4.666601107997849, + 4.781005955999717, + 4.890836943999602, + 4.9975095499976305, + 5.100563353997131, + 5.191679415998806, + 5.294081567997637, + 5.405712023999513, + 5.521628023998346, + 5.629204542998195, + 5.736299741998664, + 5.8482538539974485, + 5.96563715999946, + 6.0848996239983535, + 6.20597034399907, + 6.304353414998332, + 6.411280919997807, + 6.517208205998031, + 6.617209528998501, + 6.735191579999082, + 6.856705870999576, + 6.96243652299745, + 7.0732620309972845, + 7.184041723998234, + 7.294789587998821, + 7.3911585719979485, + 7.4966918009995425, + 7.609383988998161, + 7.731004719997145, + 7.838690852997388, + 7.947369960998913, + 8.058454243997403, + 8.159041476999846, + 8.257644786997844, + 8.367046237999602, + 8.478208459997404, + 8.600198295000155, + 8.742277762998128, + 8.845959349997429, + 8.938697318997583, + 9.027249576000031, + 9.131539540998347, + 9.249971992998326, + 9.3436184279999, + 9.44141505999869, + 9.5485278619999, + 9.638058451997495, + 9.729766927997844, + 9.826526876997377, + 9.932795650998742, + 10.04135332399892, + 10.135221169999568, + 10.24386106699967, + 10.340234295999835, + 10.447572111999762, + 10.552572684999177, + 10.667069465998793, + 10.7708711209998, + 10.871568616999866, + 10.979486245996668, + 11.077037426999595, + 11.172827194997808, + 11.261737491997337, + 11.360126808998757, + 11.464072286998999, + 11.569855424997513, + 11.6591589439995, + 11.75519763199918, + 11.85089217699715, + 11.931006270999205, + 12.01318498099863, + 12.085849201997917, + 12.151851775997784, + 12.202564311999595, + 12.261038179000025, + 12.322307949998503, + 12.369882869999856, + 12.425747039997077 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 104.11772149927856, + "p95": 121.51429100049427, + "max": 175.98400000133552 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 103.20438038145427, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 12.439041640998767, + "end_to_end_tps": 102.90181807745964, + "timing_valid": true, + "steady_tps": 110.6345993645438, + "per_stream_tps": [ + 22.0, + 22.7, + 21.3, + 23.8, + 20.6 + ], + "ttft_s": [ + 0.43, + 0.35, + 0.17, + 0.25, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.434450664997712, + "decode_tps": 22.017974744619128, + "prefill_tps": null, + "itl_mean_ms": 45.41743787059186, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.27932099852478, + "round_gap_p50_ms": 104.63495200019679, + "round_gap_p95_ms": 117.63405799865723, + "tokens_per_chunk": 2.2857142857142856 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3516730280025513, + "decode_tps": 22.712120261342953, + "prefill_tps": null, + "itl_mean_ms": 44.02935474509814, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.80379900260596, + "round_gap_p50_ms": 105.7670340014738, + "round_gap_p95_ms": 118.11346700051217, + "tokens_per_chunk": 2.392523364485981 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16628531300011673, + "decode_tps": 21.273759678273596, + "prefill_tps": null, + "itl_mean_ms": 47.00626570588165, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.48613899867632, + "round_gap_p50_ms": 104.77868499947363, + "round_gap_p95_ms": 118.71104100282537, + "tokens_per_chunk": 2.245614035087719 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2531428649999725, + "decode_tps": 23.77275403360992, + "prefill_tps": null, + "itl_mean_ms": 42.06496220783675, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.48216699884506, + "round_gap_p50_ms": 105.87692700210027, + "round_gap_p95_ms": 118.46240699742339, + "tokens_per_chunk": 2.5346534653465347 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08458569699723739, + "decode_tps": 20.640876076290855, + "prefill_tps": null, + "itl_mean_ms": 48.44755601961344, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.88391199984471, + "round_gap_p50_ms": 105.1365290004469, + "round_gap_p95_ms": 117.91115400046692, + "tokens_per_chunk": 2.1512605042016806 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 45.393115309804365, + "itl_p50_ms": 0.0, + "itl_p95_ms": 115.48613899867632, + "round_gap_p50_ms": 105.3651100010029, + "round_gap_p95_ms": 118.33047499749227, + "round_gap_mean_ms": 105.61354383211783, + "tokens_per_chunk": 2.3146473779385173, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "41f6fc95aafb", + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "99260e755b11", + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "f7167ec39d0f", + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "ba8ff1f31639", + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "a61ee957dad9", + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a" + } + ], + "client_intervals": [ + { + "start": 30487.099977957, + "first": 30487.534428622, + "last": 30499.115875279, + "end": 30499.116065981, + "elapsed_s": 12.016088024000055, + "chunk_offsets_s": [ + 0.434450664997712, + 0.5237350940005854, + 0.6348105879987997, + 0.7511256219986535, + 0.8596516549987427, + 0.9601289659985923, + 1.0514674949990876, + 1.1558098409986997, + 1.2678974150003341, + 1.3676890319984523, + 1.4665325450005184, + 1.5584690059986315, + 1.6531499259981501, + 1.7487440049990255, + 1.8610097380005755, + 1.9729762799979653, + 2.0880470970005263, + 2.2040554949999205, + 2.3147311819993774, + 2.407920330999332, + 2.5181755619996693, + 2.6198408609998296, + 2.711153205997107, + 2.8240204299981997, + 2.927401444998395, + 3.0247354909988644, + 3.133351173997653, + 3.2293962509975245, + 3.3453444189981383, + 3.452233405998413, + 3.5541525810003805, + 3.656234312999004, + 3.751756376997946, + 3.848286137999821, + 3.9493868930003373, + 4.064666213998862, + 4.167661290997785, + 4.275214663997758, + 4.384215788999427, + 4.476861003000522, + 4.57571654399726, + 4.678479947997403, + 4.786685791998025, + 4.896221593000519, + 5.002547173000494, + 5.105103825000697, + 5.196893322998221, + 5.304424826998002, + 5.419517148999148, + 5.5339471389997925, + 5.639454291998845, + 5.739681607999955, + 5.851948542000173, + 5.969490142997529, + 6.087447689998953, + 6.207305124000413, + 6.311631161999685, + 6.419698748999508, + 6.526846680997551, + 6.6271107729990035, + 6.739703278999514, + 6.859678246997646, + 6.966333541997301, + 7.077386002998537, + 7.187939088998974, + 7.305573146997631, + 7.404069255997456, + 7.509434365998459, + 7.620086967999669, + 7.732874723998975, + 7.838262995999685, + 7.94539881899982, + 8.057696471998497, + 8.159190054997453, + 8.256950513998163, + 8.373101708999457, + 8.483685682000214, + 8.60577621199991, + 8.739605287999439, + 8.844240239999635, + 8.937565538999479, + 9.028948921997653, + 9.132950038998388, + 9.250054240997997, + 9.351496398998279, + 9.450446475999343, + 9.556274334998307, + 9.644668867997098, + 9.73290339599771, + 9.827700646997982, + 9.936404861000483, + 10.04474115099947, + 10.136602466998738, + 10.244668096998794, + 10.347979619997204, + 10.455198899999232, + 10.560903803998372, + 10.673061973997392, + 10.772812636998424, + 10.872050701000262, + 10.98039809599868, + 11.078069874998619, + 11.175681380998867, + 11.268009868999798, + 11.371064311999362, + 11.474840833998314, + 11.579153464997944, + 11.668086989000585, + 11.758721285998035, + 11.854609230998904, + 11.934599164000247, + 12.015897321998636 + ], + "chunk_gaps_ms": { + "count": 111, + "p50": 104.63495200019679, + "p95": 117.63405799865723, + "max": 133.82907599952887 + } + }, + { + "start": 30487.100616956, + "first": 30487.452289984, + "last": 30498.679775444, + "end": 30498.680042251, + "elapsed_s": 11.579425294999965, + "chunk_offsets_s": [ + 0.3516730280025513, + 0.5228362449997803, + 0.6346513740027149, + 0.7502296679995197, + 0.859088184999564, + 0.9596095859997149, + 1.051352210000914, + 1.1547499700027402, + 1.267543949001265, + 1.3672717590015964, + 1.4660116449995257, + 1.5576078010017227, + 1.6524194539997552, + 1.7487215690016455, + 1.8604891100003442, + 1.9719260849997227, + 2.0877298840023286, + 2.203164722002839, + 2.3136143650008307, + 2.407548906001466, + 2.5173093680023158, + 2.618707732002804, + 2.71015002599961, + 2.8229709700026433, + 2.9265028040026664, + 3.0235912640018796, + 3.132871501002228, + 3.2286967870022636, + 3.344546493000962, + 3.451960586000496, + 3.554147560000274, + 3.6546100129999104, + 3.7512130409995734, + 3.847348701001465, + 3.94837448600083, + 4.064619151002262, + 4.167954021002515, + 4.274756500002695, + 4.3834739470003115, + 4.47575520400278, + 4.57497994699952, + 4.677425083002163, + 4.786147589002212, + 4.8956538700003875, + 5.001420904001861, + 5.104689591000351, + 5.195692469002097, + 5.304064916999778, + 5.4192343989998335, + 5.532448117999593, + 5.638887799002987, + 5.738451009001437, + 5.851073681002163, + 5.968467293001595, + 6.086580760002107, + 6.206354653000744, + 6.31130050000138, + 6.41947494499982, + 6.526506520000112, + 6.625840946002427, + 6.738548241002718, + 6.8597701040016545, + 6.966601981002896, + 7.0758738050026295, + 7.1874839320007595, + 7.305278579002334, + 7.403235412999493, + 7.508622272001958, + 7.619226099002844, + 7.732162227002846, + 7.83786401800171, + 7.944601532999513, + 8.056763128002785, + 8.157604118001473, + 8.256146528001409, + 8.372193376002542, + 8.483682181002223, + 8.60591180800111, + 8.73967223100044, + 8.843686701002298, + 8.9371054559997, + 9.028479868000431, + 9.13217535000149, + 9.249780063000799, + 9.350964385001134, + 9.45008523800061, + 9.556185534001997, + 9.643818394000846, + 9.73247843200079, + 9.827536141001474, + 9.936371935000352, + 10.04482563700003, + 10.135868941000808, + 10.244598646000668, + 10.347543685002165, + 10.454212175001885, + 10.5604299680017, + 10.673225779002678, + 10.77169073299956, + 10.871244844001922, + 10.980092813002557, + 11.076838876000693, + 11.174808710002253, + 11.267622485000174, + 11.370628841003054, + 11.474019944002066, + 11.579158488002577 + ], + "chunk_gaps_ms": { + "count": 106, + "p50": 105.9336650014302, + "p95": 118.11346700051217, + "max": 171.16321699722903 + } + }, + { + "start": 30487.100136979, + "first": 30487.266422292, + "last": 30499.253020047, + "end": 30499.25313239, + "elapsed_s": 12.15299541100103, + "chunk_offsets_s": [ + 0.16628531300011673, + 0.5235285769995244, + 0.635029803001089, + 0.7505602660021395, + 0.8597513790009543, + 0.9602266120018612, + 1.0511692330001097, + 1.1557867810006428, + 1.2682546230025764, + 1.3674133980020997, + 1.4662006680009654, + 1.5579417090011702, + 1.6528417160006939, + 1.74906038699919, + 1.8607420200023626, + 1.9723380329996871, + 2.087738360001822, + 2.2032244990004983, + 2.3143434449993947, + 2.4074236109991034, + 2.5178221280002617, + 2.6195909420021053, + 2.7109182070016686, + 2.823760660001426, + 2.9269245870018494, + 3.024032173001615, + 3.1332538160022523, + 3.2294010969999363, + 3.344638157999725, + 3.4517070530018827, + 3.554707512001187, + 3.655886108001141, + 3.7517337009994662, + 3.848930175001442, + 3.948809526002151, + 4.0646625039989885, + 4.16798237400144, + 4.274979888999951, + 4.384214014000463, + 4.476307048000308, + 4.5752599949992145, + 4.677966388000641, + 4.786413723999431, + 4.896422067002277, + 5.001801572001568, + 5.104842184002337, + 5.196786035001423, + 5.3044065489993955, + 5.41966696000236, + 5.53307878600026, + 5.638757370001258, + 5.739459819000331, + 5.851416608002182, + 5.969747082999675, + 6.08699966399945, + 6.2070641600003, + 6.311842844999774, + 6.419825116001448, + 6.5259433050014195, + 6.627135433001968, + 6.7393833640016965, + 6.860492821000662, + 6.966915659002552, + 7.076877273000719, + 7.188309171000583, + 7.305050246999599, + 7.40381477900155, + 7.508814428001642, + 7.6193853770018904, + 7.7324039740015, + 7.83817027600162, + 7.945348637000279, + 8.057288728999993, + 8.15823488100068, + 8.256481619002443, + 8.372586374000093, + 8.48393339500035, + 8.606348430999788, + 8.739767701001256, + 8.844328434999625, + 8.937504706002073, + 9.029010139001912, + 9.132175140999607, + 9.250886182002432, + 9.351213137000741, + 9.450608473001921, + 9.556588302999444, + 9.64444079399982, + 9.73304777100202, + 9.828223371001513, + 9.93657714100118, + 10.044658361999609, + 10.135991949002346, + 10.244370442000218, + 10.347635493002599, + 10.45462772099927, + 10.561195829999633, + 10.673546925001574, + 10.772289018001175, + 10.871769662000588, + 10.980463997999323, + 11.07726030699996, + 11.17497094600185, + 11.268037023000943, + 11.371253111999977, + 11.474435778000043, + 11.57911025499925, + 11.668164661001356, + 11.757648045000678, + 11.854382420002366, + 11.934537788001762, + 12.015407765000418, + 12.088089266999305, + 12.152883067999937 + ], + "chunk_gaps_ms": { + "count": 113, + "p50": 104.77868499947363, + "p95": 118.71104100282537, + "max": 357.24326399940765 + } + }, + { + "start": 30487.10107875, + "first": 30487.354221615, + "last": 30498.080786978, + "end": 30498.080937717, + "elapsed_s": 10.979858966999018, + "chunk_offsets_s": [ + 0.2531428649999725, + 0.5222916619968601, + 0.6342357959983929, + 0.7497179629972379, + 0.8586954589991365, + 0.9594845459978387, + 1.0507100129980245, + 1.1541942889998609, + 1.2663767419981014, + 1.3667509389997576, + 1.4653332109992334, + 1.5572600119994604, + 1.651734030998341, + 1.748178840996843, + 1.8601563049996912, + 1.9720185009973648, + 2.087599998998485, + 2.202343192999251, + 2.3129468849983823, + 2.4068828969975584, + 2.5167489359992032, + 2.6190892639970116, + 2.7095805589997326, + 2.8227646759987692, + 2.9259281079976063, + 3.023029569998471, + 3.1323576219983806, + 3.2283999159990344, + 3.343759138999303, + 3.4512120279978262, + 3.553171358998952, + 3.6546732769966184, + 3.7509596679992683, + 3.848037450999982, + 3.947976323997864, + 4.063846540997474, + 4.167251901999407, + 4.273768815997755, + 4.382871296998928, + 4.475227491999249, + 4.574377057997481, + 4.677252099998441, + 4.7859178359976795, + 4.895257929998479, + 5.000906752997253, + 5.103720313996746, + 5.195155257999431, + 5.3035293689972605, + 5.418654841996613, + 5.532397584996943, + 5.638149346999853, + 5.738346151996666, + 5.850528345999919, + 5.967945481999777, + 6.086407888997201, + 6.205622597997717, + 6.310948585996812, + 6.419280660997174, + 6.524548934998165, + 6.62564268099959, + 6.73801675499999, + 6.859089382996899, + 6.964966309998999, + 7.075616720998369, + 7.18713041799856, + 7.303984411999409, + 7.402706165998097, + 7.50790980599777, + 7.618393934997584, + 7.731362413996976, + 7.837039465997805, + 7.944208375996823, + 8.056248624998261, + 8.15708435399938, + 8.255631032996462, + 8.371888381996541, + 8.483515914998861, + 8.605565601999842, + 8.73913033399731, + 8.843338944996503, + 8.937055738999334, + 9.028325483999652, + 9.131336484999338, + 9.249701943997934, + 9.350331095000001, + 9.449271256999054, + 9.555486789999122, + 9.64299507299802, + 9.731903702999261, + 9.826842976999615, + 9.935683904997859, + 10.043345116999262, + 10.134616112998629, + 10.244041284997365, + 10.346538377998513, + 10.453607333998661, + 10.560482933997264, + 10.672091305998038, + 10.7713062599978, + 10.870732595998561, + 10.979708227998344 + ], + "chunk_gaps_ms": { + "count": 100, + "p50": 106.04623000108404, + "p95": 118.46240699742339, + "max": 269.14879699688754 + } + }, + { + "start": 30487.099884677, + "first": 30487.184470374, + "last": 30499.538597159, + "end": 30499.538926318, + "elapsed_s": 12.439041640998767, + "chunk_offsets_s": [ + 0.08458569699723739, + 0.5233672359972843, + 0.6346089639991987, + 0.7511636029994406, + 0.8596546449989546, + 0.9599924449976243, + 1.0511643229983747, + 1.1563008519988216, + 1.2679450869982247, + 1.3678377389987872, + 1.4666745679969608, + 1.5579572209971957, + 1.6527247999983956, + 1.748738215999765, + 1.8608037639969552, + 1.9725099919996865, + 2.0875061429978814, + 2.2036203749994456, + 2.3139813439993304, + 2.40755112899933, + 2.518462213996827, + 2.620375401998899, + 2.711061381000036, + 2.824507570996502, + 2.927550999997038, + 3.0250846079979965, + 3.133349718998943, + 3.229338104996714, + 3.3452220169965585, + 3.451752820998081, + 3.553966452996974, + 3.6556423619986163, + 3.752038727998297, + 3.848859491998155, + 3.9494220909982687, + 4.0646581379987765, + 4.167990688998543, + 4.274838217999786, + 4.384649005998654, + 4.476801752996835, + 4.576040393996664, + 4.678621922997991, + 4.787015605998022, + 4.896853683996596, + 5.002539847999287, + 5.105366073999903, + 5.196933734998311, + 5.304587446997175, + 5.419699534999381, + 5.5338274569985515, + 5.639254969999456, + 5.739447983996797, + 5.8519105139967, + 5.9698216679971665, + 6.08746691199849, + 6.2069847129969276, + 6.3125093819980975, + 6.4206297589989845, + 6.527469080996525, + 6.626759720998962, + 6.7388720419985475, + 6.859678821998386, + 6.966759626997373, + 7.077398364999681, + 7.187980682996567, + 7.305832117999671, + 7.40364774399859, + 7.5089655659976415, + 7.620042878999811, + 7.732730574000016, + 7.838481938997575, + 7.945108774998516, + 8.058016690996737, + 8.159238493997691, + 8.256582366997463, + 8.37258897999709, + 8.484092128997872, + 8.60581691099651, + 8.739629211999272, + 8.844794451997586, + 8.938004512998305, + 9.02898578299937, + 9.13327361699703, + 9.250314506996801, + 9.351768946999073, + 9.45092909199957, + 9.556218107998575, + 9.6446301199976, + 9.73274355799731, + 9.827820280999731, + 9.936206481997942, + 10.045454192997568, + 10.136525290999998, + 10.245151224997244, + 10.34797352699752, + 10.454620859996794, + 10.561069829996995, + 10.673025239997514, + 10.772045525998692, + 10.872180474998459, + 10.980226678999315, + 11.077751333999913, + 11.1753772009979, + 11.26822139199794, + 11.37108323099892, + 11.474520677998953, + 11.579671582996525, + 11.668035574999521, + 11.758111342998745, + 11.85441317099685, + 11.934531502000027, + 12.015431079998962, + 12.088181586997962, + 12.152815869998449, + 12.204054537996853, + 12.26478408499679, + 12.331646729999193, + 12.381801536997955, + 12.438712481998664 + ], + "chunk_gaps_ms": { + "count": 118, + "p50": 105.14371699900948, + "p95": 117.91115400046692, + "max": 438.7815390000469 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "41f6fc95aafbcc77b894251249763e05ea1fcaa6dbf1b34fe5fdd8029e2aa84c", + "output_sha256": "d1507a24293327c18d65a16b86ce9708b46245214e0596c2ea5759da8efe82d8", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "99260e755b117f3c52753835c802fa66936daec43205142c0e34c4d9baf5e518", + "output_sha256": "6aea89cc3b9c50fd53ae3e0a8e54cedd351745af75aea4335f4042f3c3c5e894", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "f7167ec39d0f02879f59c87d4c914a059e8c657110e1691efc8271448878422d", + "output_sha256": "f68e1d916d1e9e0f9364ea47dba5aad3c546ae7b41239ec0a9c71f3e0bbe7838", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ba8ff1f316397a9f2dc4185e809cbb7155828ab0825e7d08f067909cb88dfa63", + "output_sha256": "d4c784f6d9c574d6779e0b9c9eb28134a46927c951e77c53329a6fff71643b0b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a61ee957dad9fb6d2d190506d2d787619706fcd4d23bde9ed9190d309ef390bb", + "output_sha256": "bc198dd64f79fa6533e1bff07a9af27fb0fdeb38b5717e818cb117f17743e07a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.2, + "per_stream_tps": 67.2, + "ttft_s_max": 0.1, + "request": { + "ttft_s": 0.096, + "decode_tps": 67.184, + "prefill_tps": 146.571, + "itl_mean_ms": 14.885, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.468, + "round_gap_p50_ms": 73.114, + "round_gap_p95_ms": 80.153, + "tokens_per_chunk": 4.571 + }, + "pooled": { + "aggregate_prefill_tps": 146.571, + "itl_mean_ms": 14.885, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.468, + "round_gap_p50_ms": 73.114, + "round_gap_p95_ms": 80.153, + "tokens_per_chunk": 4.571 + }, + "timing_valid": true, + "end_to_end_tps": 65.78820328142699, + "makespan_s": 3.891275140998914, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.30877172518886, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.8848132109997096, + "end_to_end_tps": 65.89763422219251, + "timing_valid": true, + "per_stream_tps": [ + 67.3 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09619021300022723, + "decode_tps": 67.30877172518886, + "prefill_tps": 145.5449526862668, + "itl_mean_ms": 14.856904596073196, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.46797200161382, + "round_gap_p50_ms": 72.38873499954934, + "round_gap_p95_ms": 80.15315200100304, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.856904596073196, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.46797200161382, + "round_gap_p50_ms": 72.38873499954934, + "round_gap_p95_ms": 80.15315200100304, + "round_gap_mean_ms": 68.88201221815756, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 145.5449526862668 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30513.815434696, + "first": 30513.911624909, + "last": 30517.700135581, + "end": 30517.700247907, + "elapsed_s": 3.8848132109997096, + "chunk_offsets_s": [ + 0.09619021300022723, + 0.15005343500160961, + 0.20435601699864492, + 0.25436090199946193, + 0.30919158200049424, + 0.37637132500094594, + 0.4557658000012452, + 0.5275578949986084, + 0.6066870009999548, + 0.6829298660013592, + 0.7553186010009085, + 0.8273467200015148, + 0.9016710280011466, + 0.9797085799982597, + 1.0558635980014515, + 1.133185989001504, + 1.2103340520006896, + 1.2890299350001442, + 1.3600973219981825, + 1.416411751000851, + 1.4748271239986934, + 1.5354840899999544, + 1.6000994550013274, + 1.679107561998535, + 1.741606118001073, + 1.7909581109997816, + 1.8497727370013308, + 1.9253351929983182, + 2.0021645599990734, + 2.0700845700011996, + 2.1199735609989148, + 2.1827106929995352, + 2.2586101400011103, + 2.337467088000267, + 2.41762024000127, + 2.4984433380013797, + 2.5831557669989706, + 2.6608287259987264, + 2.7361323650002305, + 2.8117068909996306, + 2.887646813000174, + 2.966402100999403, + 3.043098801001179, + 3.1192296619992703, + 3.196697634000884, + 3.2641591989995504, + 3.3185261719991104, + 3.382241357998282, + 3.4601369359988894, + 3.5258287149990792, + 3.5928327699984948, + 3.6674256139995123, + 3.731064071998844, + 3.776762677000079, + 3.8263811090000672, + 3.8847008849988924 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 72.38873499954934, + "p95": 80.15315200100304, + "max": 84.7124289975909 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.06173890694781, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.89771570800076, + "end_to_end_tps": 65.67949516546682, + "timing_valid": true, + "per_stream_tps": [ + 67.1 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09492854900236125, + "decode_tps": 67.06173890694781, + "prefill_tps": 147.47934259114993, + "itl_mean_ms": 14.91163241960606, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.36153300013393, + "round_gap_p50_ms": 74.17465199978324, + "round_gap_p95_ms": 80.27183500234969, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.91163241960606, + "itl_p50_ms": 0.0, + "itl_p95_ms": 78.36153300013393, + "round_gap_p50_ms": 74.17465199978324, + "round_gap_p95_ms": 80.27183500234969, + "round_gap_mean_ms": 69.13575030908264, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 147.47934259114993 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30517.70125082, + "first": 30517.796179369, + "last": 30521.598645636, + "end": 30521.598966528, + "elapsed_s": 3.89771570800076, + "chunk_offsets_s": [ + 0.09492854900236125, + 0.15031844900295255, + 0.20596656999987317, + 0.2563099040016823, + 0.3092552730013267, + 0.37654554300024756, + 0.45520269600092433, + 0.5293773480007076, + 0.6035035770000832, + 0.6837754120024329, + 0.7606449470004009, + 0.8327177419996588, + 0.9074811020000197, + 0.9863424550021591, + 1.061670328002947, + 1.1386792880002758, + 1.216783182000654, + 1.2959184060018742, + 1.366223599001387, + 1.4209130960007315, + 1.479970256001252, + 1.5398149190004915, + 1.5998220920009771, + 1.680763709002349, + 1.7479862479995063, + 1.7986173940007575, + 1.856865990001097, + 1.93357349100188, + 2.0103104330009955, + 2.07582264300072, + 2.126183773001685, + 2.1901723830014816, + 2.269078695000644, + 2.3459352030004084, + 2.4254152390021773, + 2.5053518429995165, + 2.5865662440010055, + 2.664814923002268, + 2.7449595810030587, + 2.8215833280009974, + 2.8971940539995558, + 2.9755555869996897, + 3.052170250000927, + 3.1276805050001713, + 3.2068904820007447, + 3.2758286780008348, + 3.329479166000965, + 3.3936984520005353, + 3.4722886640010984, + 3.537111225999979, + 3.5975506440008758, + 3.6733744560006016, + 3.740787620001356, + 3.789167926002847, + 3.839605985001981, + 3.8973948160019063 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 74.17465199978324, + "p95": 80.27183500234969, + "max": 81.21440100148902 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 67.18354786768077, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 3.891275140998914, + "end_to_end_tps": 65.78820328142699, + "timing_valid": true, + "per_stream_tps": [ + 67.2 + ], + "ttft_s": [ + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09551681399898371, + "decode_tps": 67.18354786768077, + "prefill_tps": 146.57105292633565, + "itl_mean_ms": 14.884596478434247, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.4398210014624, + "round_gap_p50_ms": 73.11440400007996, + "round_gap_p95_ms": 79.94784699985757, + "tokens_per_chunk": 4.571428571428571 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 14.884596478434247, + "itl_p50_ms": 0.0, + "itl_p95_ms": 77.4398210014624, + "round_gap_p50_ms": 73.11440400007996, + "round_gap_p95_ms": 79.94784699985757, + "round_gap_mean_ms": 69.01040185455878, + "tokens_per_chunk": 4.571428571428571, + "aggregate_prefill_tps": 146.57105292633565 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30521.600293725, + "first": 30521.695810539, + "last": 30525.491382641, + "end": 30525.491568866, + "elapsed_s": 3.891275140998914, + "chunk_offsets_s": [ + 0.09551681399898371, + 0.14917676899858634, + 0.2023242039977049, + 0.2536992489986005, + 0.30879400199773954, + 0.37781888300014543, + 0.45655226499729906, + 0.5271430849970784, + 0.6003106429998297, + 0.6753948730001866, + 0.7500924369996937, + 0.827425666997442, + 0.9051694399968255, + 0.9832929719996173, + 1.0602871749979386, + 1.136703079999279, + 1.213355011997919, + 1.2933028589977766, + 1.3653545619999932, + 1.4212023629988835, + 1.4791624179997598, + 1.5385775399990962, + 1.5974675379984546, + 1.672674179997557, + 1.7341928409987304, + 1.7895785729997442, + 1.8525461279969022, + 1.92996639699777, + 2.0073107030002575, + 2.0721316679992015, + 2.12176203999843, + 2.1852663939971535, + 2.2637488359978306, + 2.341188656999293, + 2.420956860998558, + 2.5012201739991724, + 2.5806706519979343, + 2.657496337000339, + 2.730610741000419, + 2.811935417998029, + 2.8911419399992155, + 2.9707510089974676, + 3.0475667469982, + 3.124408487998153, + 3.2015480310001294, + 3.2713601069990546, + 3.3255291919995216, + 3.389220987999579, + 3.467937608998909, + 3.5318425449986535, + 3.592887573999178, + 3.665990169996803, + 3.7290689649998967, + 3.780002814997715, + 3.832556208999449, + 3.891088915999717 + ], + "chunk_gaps_ms": { + "count": 55, + "p50": 73.11440400007996, + "p95": 79.94784699985757, + "max": 81.32467699761037 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.4, + "per_stream_tps": 62.8, + "ttft_s_max": 0.19, + "request": { + "ttft_s": 0.141, + "decode_tps": 62.821, + "prefill_tps": 110.628, + "itl_mean_ms": 15.92, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.91, + "round_gap_p50_ms": 76.678, + "round_gap_p95_ms": 85.715, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 148.308, + "itl_mean_ms": 15.894, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.836, + "round_gap_p50_ms": 77.203, + "round_gap_p95_ms": 85.653, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 121.99932686545984, + "makespan_s": 4.196744466997188, + "steady_tps": 126.3, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 123.96046592248003, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.208981826999661, + "end_to_end_tps": 121.64462120402527, + "timing_valid": true, + "steady_tps": 125.86018218133772, + "per_stream_tps": [ + 63.4, + 62.0 + ], + "ttft_s": [ + 0.19, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18870432100084145, + "decode_tps": 63.430940056281116, + "prefill_tps": 74.19014003361148, + "itl_mean_ms": 15.765177043138857, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.83612799964612, + "round_gap_p50_ms": 75.62324299942702, + "round_gap_p95_ms": 85.39511399794719, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09470166300161509, + "decode_tps": 61.987540569985825, + "prefill_tps": 147.83267322096802, + "itl_mean_ms": 16.13227417646889, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.86231500055874, + "round_gap_p50_ms": 77.30728699971223, + "round_gap_p95_ms": 86.03410700015957, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.948725609803873, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.83612799964612, + "round_gap_p50_ms": 77.20286000039778, + "round_gap_p95_ms": 85.65332399666659, + "round_gap_mean_ms": 72.62366125892835, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 148.30787337212817 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30525.492404777, + "first": 30525.681109098, + "last": 30529.701229244, + "end": 30529.701294475, + "elapsed_s": 4.208889697998529, + "chunk_offsets_s": [ + 0.18870432100084145, + 0.24368302700167987, + 0.2996038460005366, + 0.35316096099995775, + 0.40643596299923956, + 0.4743521480013442, + 0.5524952150008176, + 0.6281184580002446, + 0.7056360189999396, + 0.7857247850006388, + 0.8637521189994004, + 0.9461686620015826, + 1.026744859998871, + 1.1074471380015893, + 1.1863499630017031, + 1.2668249650014332, + 1.3490013640002871, + 1.4318272939999588, + 1.5072120600016206, + 1.5678867809983785, + 1.6195692549990781, + 1.6813283849987783, + 1.7503204310014553, + 1.829210233001504, + 1.9148635569981707, + 1.9849159129989857, + 2.036919535999914, + 2.0906137079982727, + 2.155822332999378, + 2.233630364000419, + 2.300734779000777, + 2.3547668979990704, + 2.41375987200081, + 2.4909627320012078, + 2.5709966819995316, + 2.6548082970002724, + 2.741652739001438, + 2.827047852999385, + 2.9122017129993765, + 2.9952037389994075, + 3.073959442001069, + 3.154047284999251, + 3.235871300999861, + 3.316835629000707, + 3.3974355409991404, + 3.4787552790003247, + 3.551697224000236, + 3.6047341429984954, + 3.6687839170008374, + 3.7496200450004835, + 3.818399112999032, + 3.879141654000705, + 3.9495876030014188, + 4.024509775001206, + 4.086302070998499, + 4.138302332998137, + 4.20882446700125 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.4130514999124, + "p95": 85.39511399794719, + "max": 86.84444200116559 + } + }, + { + "start": 30525.492312648, + "first": 30525.587014311, + "last": 30529.700744226, + "end": 30529.7008586, + "elapsed_s": 4.208545952002169, + "chunk_offsets_s": [ + 0.09470166300161509, + 0.2436300770023081, + 0.2999414180012536, + 0.3527822890027892, + 0.4059544960000494, + 0.47417735700219055, + 0.5523954090022016, + 0.6284589570022945, + 0.7059534740001254, + 0.7856247080017056, + 0.8639253110013669, + 0.9459301910028444, + 1.026748314001452, + 1.1071205680018466, + 1.186264965999726, + 1.266537479001272, + 1.3488360880000982, + 1.4314939430005325, + 1.5070879790000618, + 1.567455415999575, + 1.6195082779995573, + 1.6819198760022118, + 1.7502144850004697, + 1.829108059999271, + 1.9151421669994306, + 1.9852165280026384, + 2.0371322600003623, + 2.090396346000489, + 2.155586853001296, + 2.233933089999482, + 2.3006110490023275, + 2.354372890000377, + 2.4139417320002394, + 2.4912490189999517, + 2.571417076000216, + 2.6547280170016165, + 2.7416175239995937, + 2.82745450999937, + 2.912497290999454, + 2.9948509740024747, + 3.07398432500122, + 3.153923682999448, + 3.235800857000868, + 3.3166631720014266, + 3.397285905000899, + 3.4785095160004857, + 3.551618976001919, + 3.60449471299944, + 3.668741522000346, + 3.7495087730021623, + 3.818731631999981, + 3.879106120002689, + 3.9494470019999426, + 4.02402486900246, + 4.086647575000825, + 4.138173949999327, + 4.208431578001182 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 77.40090199877159, + "p95": 86.03410700015957, + "max": 148.928414000693 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.3639784121187, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.196744466997188, + "end_to_end_tps": 121.99932686545984, + "timing_valid": true, + "steady_tps": 126.3168212300856, + "per_stream_tps": [ + 63.7, + 62.2 + ], + "ttft_s": [ + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19107395900209667, + "decode_tps": 63.66316889675108, + "prefill_tps": 73.27005769449922, + "itl_mean_ms": 15.70766924313491, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.64993799780495, + "round_gap_p50_ms": 75.69424500252353, + "round_gap_p95_ms": 86.37721599734505, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09577199899649713, + "decode_tps": 62.18732146178795, + "prefill_tps": 146.18051358113607, + "itl_mean_ms": 16.08044817647383, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.01464900028077, + "round_gap_p50_ms": 76.18497899966314, + "round_gap_p95_ms": 86.43561800272437, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.894058709804371, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.64993799780495, + "round_gap_p50_ms": 76.18497899966314, + "round_gap_p95_ms": 86.37721599734505, + "round_gap_mean_ms": 72.37473162500204, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 146.4571063543662 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30529.702259601, + "first": 30529.89333356, + "last": 30533.898789217, + "end": 30533.898895771, + "elapsed_s": 4.196636169999692, + "chunk_offsets_s": [ + 0.19107395900209667, + 0.24703704700004891, + 0.30207530800180393, + 0.3543758390005678, + 0.4060477290004201, + 0.47496308799964027, + 0.5531529050022073, + 0.6280387840015464, + 0.7118332440004451, + 0.792508695001743, + 0.8695367500004068, + 0.9463214410025103, + 1.0260285200020007, + 1.107465763001528, + 1.1883062119995884, + 1.2686495060006564, + 1.3507003160011664, + 1.4313502539989713, + 1.5070444990014948, + 1.5651476269995328, + 1.617986771001597, + 1.6842450389995065, + 1.7545314050003071, + 1.8328679090009246, + 1.912303344000975, + 1.9804466700006742, + 2.03265982800076, + 2.0865239720005775, + 2.153804816000047, + 2.2318639340010122, + 2.297792728000786, + 2.350246297999547, + 2.410864331999619, + 2.487629403000028, + 2.5697834660022636, + 2.6534066240019456, + 2.7452586140025232, + 2.8316358299998683, + 2.9099492989989812, + 2.9901461320005183, + 3.0691007470013574, + 3.1485196470021037, + 3.2307009540018043, + 3.310581092002394, + 3.389610862999689, + 3.470444710001175, + 3.5422977220005123, + 3.595665138000186, + 3.6618613400023605, + 3.7482675749997725, + 3.8196938760011108, + 3.8752978689990414, + 3.9390948090003803, + 4.0121370410015516, + 4.075050845000078, + 4.125252911999269, + 4.196529616001499 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.2296580014663, + "p95": 86.37721599734505, + "max": 91.85199000057764 + } + }, + { + "start": 30529.702151304, + "first": 30529.797923303, + "last": 30533.898437588, + "end": 30533.898520362, + "elapsed_s": 4.196369057997799, + "chunk_offsets_s": [ + 0.09577199899649713, + 0.24694131899741478, + 0.3017721019969031, + 0.35408061299676774, + 0.4059619809995638, + 0.4749956949999614, + 0.5531605819996912, + 0.6282010859977163, + 0.7119765949973953, + 0.7930982199977734, + 0.8692831989974366, + 0.9463113179990614, + 1.0259821439976804, + 1.1072810729965568, + 1.1882957219968375, + 1.2684175109970965, + 1.3505547419990762, + 1.4316437369998312, + 1.5070811209989188, + 1.5655958149982325, + 1.617843099997117, + 1.6845608939984231, + 1.7545208349984023, + 1.8334668689967657, + 1.9124958379979944, + 1.9803815349987417, + 2.0330711389979115, + 2.086504924998735, + 2.1536611769988667, + 2.231574624998757, + 2.297776608997083, + 2.3502153219997126, + 2.4108230089987046, + 2.487175552996632, + 2.569656634997955, + 2.653588803997991, + 2.7451483339973493, + 2.8315839520000736, + 2.910532058998797, + 2.990374511999107, + 3.068919494999136, + 3.148393550000037, + 3.2312162759990315, + 3.3104520379965834, + 3.3894887489987013, + 3.4700057319969346, + 3.542162637997535, + 3.5954833419964416, + 3.6620093769997766, + 3.7482390779987327, + 3.8195299909966707, + 3.8751489839996793, + 3.939108225997188, + 4.011799244999565, + 4.074826430998655, + 4.125434707999375, + 4.196286283997324 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 76.26876149879536, + "p95": 86.43561800272437, + "max": 151.16932000091765 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 124.40801167268526, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 4.193427678001171, + "end_to_end_tps": 122.09582215664886, + "timing_valid": true, + "steady_tps": 126.2789695481685, + "per_stream_tps": [ + 63.6, + 62.2 + ], + "ttft_s": [ + 0.19, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.18647813300049165, + "decode_tps": 63.64275198691516, + "prefill_tps": 75.07582671885228, + "itl_mean_ms": 15.7127083411729, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.9584119997453, + "round_gap_p50_ms": 77.1711220004363, + "round_gap_p95_ms": 83.50979699753225, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09389926000221749, + "decode_tps": 62.21062360960748, + "prefill_tps": 149.09595666323017, + "itl_mean_ms": 16.074424945091938, + "itl_p50_ms": 0.0, + "itl_p95_ms": 81.11390100020799, + "round_gap_p50_ms": 77.31747899742913, + "round_gap_p95_ms": 84.43817399893305, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 15.89356664313242, + "itl_p50_ms": 0.0, + "itl_p95_ms": 80.99968399983482, + "round_gap_p50_ms": 77.31747899742913, + "round_gap_p95_ms": 84.0061050002987, + "round_gap_mean_ms": 72.37249096426369, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 150.0752437069177 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30533.899813178, + "first": 30534.086291311, + "last": 30538.093031938, + "end": 30538.093145912, + "elapsed_s": 4.19333273400116, + "chunk_offsets_s": [ + 0.18647813300049165, + 0.24141450300157885, + 0.2971698490000563, + 0.35130241899969405, + 0.40535554100279114, + 0.47639812499983236, + 0.55844109499958, + 0.6356122170000162, + 0.7123299850027252, + 0.7898038500024995, + 0.8677709200019308, + 0.9439592670023558, + 1.0252202750016295, + 1.1065084540023236, + 1.1857037900008436, + 1.265984830999514, + 1.3479935400027898, + 1.4291415010011406, + 1.5102676179994887, + 1.5730377809995844, + 1.6260407850022602, + 1.687662251002621, + 1.7550868530015578, + 1.83264194900039, + 1.9133177520016034, + 1.9815210230008233, + 2.0347386930006905, + 2.0869733250001445, + 2.152777588002209, + 2.230333364001126, + 2.295339791002334, + 2.3494666639999195, + 2.4105674880011065, + 2.489862334001373, + 2.5744899160017667, + 2.657999712999299, + 2.743317600001319, + 2.8265524480011663, + 2.906367305000458, + 2.987869712000247, + 3.0670537750011135, + 3.146370234000642, + 3.227529088002484, + 3.3076345230001607, + 3.386727224002243, + 3.4676856360019883, + 3.5455575960004353, + 3.6002252559992485, + 3.6633116560005874, + 3.7441386450009304, + 3.8129574510021484, + 3.8695719000024837, + 3.9353800940007204, + 4.010342448000301, + 4.073392809001234, + 4.12347962900094, + 4.193218759999581 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 77.32249350010534, + "p95": 83.50979699753225, + "max": 85.31788700202014 + } + }, + { + "start": 30533.899718234, + "first": 30533.993617494, + "last": 30538.092595855, + "end": 30538.092670009, + "elapsed_s": 4.192951774999528, + "chunk_offsets_s": [ + 0.09389926000221749, + 0.24159669900109293, + 0.297130750001088, + 0.3511184810013219, + 0.40562802700151224, + 0.47671553200052585, + 0.5582671360025415, + 0.6355846149999707, + 0.7127042600004643, + 0.7897979490007856, + 0.8674792759993579, + 0.9439513350007473, + 1.0251861489996372, + 1.106185832999472, + 1.1856756119996135, + 1.265914083000098, + 1.3477555490026134, + 1.4291362240001035, + 1.510623629001202, + 1.5734328120015562, + 1.6256147460007924, + 1.68754629400064, + 1.754727114999696, + 1.8328319160027604, + 1.9136260760024015, + 1.9814129500009585, + 2.0346000589997857, + 2.0868427509994945, + 2.1526029400010884, + 2.230354185001488, + 2.2952600240023457, + 2.349868011999206, + 2.4102976730027876, + 2.4900091720010096, + 2.5744473459999426, + 2.6584534510002413, + 2.7433118270018895, + 2.8267724809993524, + 2.9065400649997173, + 2.9876539659999253, + 3.066720440001518, + 3.146209356000327, + 3.2278958300012164, + 3.307406063002418, + 3.386530620002304, + 3.4677177949997713, + 3.545844489999581, + 3.6000836480016005, + 3.6631083190004574, + 3.7439785030001076, + 3.812965927001642, + 3.8693780310022703, + 3.9352450420010427, + 4.010504542002309, + 4.073255711002275, + 4.123343761999422, + 4.192877621000662 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 77.49940299800073, + "p95": 84.43817399893305, + "max": 147.69743899887544 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 210.4, + "per_stream_tps": 54.3, + "ttft_s_max": 0.39, + "request": { + "ttft_s": 0.239, + "decode_tps": 54.291, + "prefill_tps": 60.898, + "itl_mean_ms": 18.42, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.485, + "round_gap_p50_ms": 87.307, + "round_gap_p95_ms": 98.734, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 145.687, + "itl_mean_ms": 18.448, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.466, + "round_gap_p50_ms": 87.63, + "round_gap_p95_ms": 97.504, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 207.08491288614647, + "makespan_s": 4.944831497999985, + "steady_tps": 220.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 211.60334357185494, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.917022502999316, + "end_to_end_tps": 208.2561142185894, + "timing_valid": true, + "steady_tps": 222.35046920647125, + "per_stream_tps": [ + 54.0, + 55.1, + 56.4, + 52.9 + ], + "ttft_s": [ + 0.19, + 0.29, + 0.39, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19264569499864592, + "decode_tps": 53.98552533740787, + "prefill_tps": 72.67227020099465, + "itl_mean_ms": 18.52348372549921, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.17260399964289, + "round_gap_p50_ms": 87.20238499881816, + "round_gap_p95_ms": 98.60767600184772, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2905599039986555, + "decode_tps": 55.13173157020954, + "prefill_tps": 48.182835302921845, + "itl_mean_ms": 18.13837460785198, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.85500500097987, + "round_gap_p50_ms": 87.09111400094116, + "round_gap_p95_ms": 98.86073099914938, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.39147257800141233, + "decode_tps": 56.36837251708493, + "prefill_tps": 35.76240274982809, + "itl_mean_ms": 17.740444780393577, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.13311900020926, + "round_gap_p50_ms": 87.07118800157332, + "round_gap_p95_ms": 97.48843900160864, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09642912200069986, + "decode_tps": 52.909089118209856, + "prefill_tps": 145.18435623522933, + "itl_mean_ms": 18.90034428235559, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.99300500040408, + "round_gap_p50_ms": 86.60551799766836, + "round_gap_p95_ms": 99.10676700019394, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.32566184902509, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.89679100018111, + "round_gap_p50_ms": 87.20238499881816, + "round_gap_p95_ms": 97.50351899856469, + "round_gap_mean_ms": 83.4472102053821, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 142.56068318814204 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30538.094927201, + "first": 30538.287572896, + "last": 30543.011061246, + "end": 30543.01113078, + "elapsed_s": 4.916203578999557, + "chunk_offsets_s": [ + 0.19264569499864592, + 0.45481998800096335, + 0.5154653760000656, + 0.5722185769991484, + 0.6293601249999483, + 0.7062832150004397, + 0.7945392250003351, + 0.8843814870015194, + 0.9760879559980822, + 1.0626626900011615, + 1.1499437060010678, + 1.2374147049995372, + 1.336022381001385, + 1.4292571480000333, + 1.5184996649986715, + 1.6083873120005592, + 1.6997333250001248, + 1.7927166769986798, + 1.8763003749991185, + 1.9441743370007316, + 2.001639123998757, + 2.0677776140000788, + 2.1432219320013246, + 2.235025976999168, + 2.332274526997935, + 2.413430239997979, + 2.4726041940011783, + 2.530103955999948, + 2.6016916679982387, + 2.687520580999262, + 2.764382059998752, + 2.822787512999639, + 2.8879687180015026, + 2.9751711030003207, + 3.0673437069999636, + 3.1595328619987413, + 3.25638699800038, + 3.360975391999091, + 3.454294087001472, + 3.545371037998848, + 3.633969115999207, + 3.7229229689983185, + 3.8169280779984547, + 3.906923521000863, + 3.996675200000027, + 4.085974164001527, + 4.168465615999594, + 4.22692026599907, + 4.299386587998015, + 4.395382623999467, + 4.476713511998241, + 4.540971253998578, + 4.613113899999007, + 4.696780341000704, + 4.766324837000866, + 4.8232498559991654, + 4.916134045000945 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.2417004993622, + "p95": 98.60767600184772, + "max": 262.1742930023174 + } + }, + { + "start": 30538.095558833, + "first": 30538.386118737, + "last": 30543.011404262, + "end": 30543.011657932, + "elapsed_s": 4.916099098998529, + "chunk_offsets_s": [ + 0.2905599039986555, + 0.45398364400170976, + 0.5143899210015661, + 0.5725121079995006, + 0.6285192229988752, + 0.7060559220008145, + 0.7934291470010066, + 0.8839233670005342, + 0.9746409129984386, + 1.0619641800003592, + 1.1487548840013915, + 1.2364703200000804, + 1.3353310509992298, + 1.4289949769990926, + 1.5178312839998398, + 1.6076242109993473, + 1.6992504820009344, + 1.7919071830001485, + 1.876184065000416, + 1.9436806990015612, + 2.0011325650011713, + 2.067347623000387, + 2.1424222409987124, + 2.2345637469989015, + 2.331487758001458, + 2.412667346001399, + 2.471944672000973, + 2.530636838000646, + 2.600990183000249, + 2.687123806001182, + 2.7633852690014464, + 2.8224262399999134, + 2.8877397139985987, + 2.97483082799954, + 3.066579181999259, + 3.158434187000239, + 3.2559377059988037, + 3.360650421000173, + 3.4541152019992296, + 3.544574993000424, + 3.633085564000794, + 3.7224193200017908, + 3.8163501150011143, + 3.907001785999455, + 3.995891405000293, + 4.085034656000062, + 4.167626744998415, + 4.226006089000293, + 4.298647938001523, + 4.39446027500162, + 4.476419005000935, + 4.540366489000007, + 4.612252832001104, + 4.6956833449985425, + 4.766055424999649, + 4.822734104000119, + 4.91584542900091 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.20719050143089, + "p95": 98.86073099914938, + "max": 163.42374000305426 + } + }, + { + "start": 30538.095978028, + "first": 30538.487450606, + "last": 30543.011264025, + "end": 30543.01146714, + "elapsed_s": 4.915489112001524, + "chunk_offsets_s": [ + 0.39147257800141233, + 0.45371806700131856, + 0.5142105080012698, + 0.5720200380019378, + 0.6285712930002774, + 0.7059044000016002, + 0.7935884910002642, + 0.8833904220009572, + 0.9743171580012131, + 1.0617324100021506, + 1.1489500179995957, + 1.2363125600022613, + 1.3350611410023703, + 1.4283225660001335, + 1.5174817020015325, + 1.607265130001906, + 1.698739220999414, + 1.7916285720020824, + 1.8758975230011856, + 1.9436935250014358, + 2.000563813999179, + 2.067016576002061, + 2.1420993650026503, + 2.23434077300044, + 2.330793278000783, + 2.4121998240007088, + 2.471471376000409, + 2.529714298001636, + 2.6000849150004797, + 2.686353125001915, + 2.7633916980012145, + 2.822175632001745, + 2.886771981000493, + 2.973843169002066, + 3.0659762880022754, + 3.158201984999323, + 3.2556904240009317, + 3.3602916430027108, + 3.453506551002647, + 3.544275561001996, + 3.632794847999321, + 3.721927474001859, + 3.8161077100012335, + 3.9063516999995045, + 3.995800444001361, + 4.085148071000731, + 4.1672794180012716, + 4.226179794000927, + 4.29838936500164, + 4.394235956999182, + 4.476203898000676, + 4.53982148599971, + 4.612197788999765, + 4.69538188200022, + 4.765461105002032, + 4.822464493001462, + 4.915285997001774 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.14439799950924, + "p95": 97.48843900160864, + "max": 104.6012190017791 + } + }, + { + "start": 30538.094635429, + "first": 30538.191064551, + "last": 30543.010652343, + "end": 30543.010765071, + "elapsed_s": 4.916129642002488, + "chunk_offsets_s": [ + 0.09642912200069986, + 0.45500968399937847, + 0.5154644480026036, + 0.5724464609993447, + 0.6291598099996918, + 0.7067591980012367, + 0.7942842340016796, + 0.8846171270015475, + 0.9759689759994217, + 1.0631965630018385, + 1.1498020809995069, + 1.2373609080023016, + 1.3364676750024955, + 1.4296311660000356, + 1.5186868980017607, + 1.6088333900006546, + 1.699969109002268, + 1.7929097630003525, + 1.8765270760013664, + 1.9443405880010687, + 2.0018551270004536, + 2.0684725570026785, + 2.1432624389999546, + 2.2352554440003587, + 2.3326492010019138, + 2.413299394000205, + 2.472580541001662, + 2.5313558810012182, + 2.6019586680013163, + 2.687907040999562, + 2.7642090910012485, + 2.8229997570015257, + 2.8883972850017017, + 2.974954589000845, + 3.067716638001002, + 3.1596134290011832, + 3.2565831860010803, + 3.3613299960015866, + 3.4545062510005664, + 3.54541672599953, + 3.634550709000905, + 3.7234484149994387, + 3.817058412001643, + 3.9073551900000894, + 3.997062043999904, + 4.086089865002577, + 4.1686754140027915, + 4.227422842999658, + 4.299497011001222, + 4.395199916001729, + 4.477242875000229, + 4.5409973050009285, + 4.613018796000688, + 4.696402307999961, + 4.76707252900087, + 4.823442647000775, + 4.916016914001375 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 86.9165525000426, + "p95": 99.10676700019394, + "max": 358.5805619986786 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 209.92746876291432, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.956782362001832, + "end_to_end_tps": 206.58562858233913, + "timing_valid": true, + "steady_tps": 220.0038481284867, + "per_stream_tps": [ + 52.5, + 54.6, + 55.8, + 53.5 + ], + "ttft_s": [ + 0.1, + 0.28, + 0.38, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09715656600019429, + "decode_tps": 52.4860694500246, + "prefill_tps": 144.09731196110826, + "itl_mean_ms": 19.052674556858655, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.5531510001747, + "round_gap_p50_ms": 87.49592800086248, + "round_gap_p95_ms": 101.48921699874336, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.2849944699992193, + "decode_tps": 54.59549071123958, + "prefill_tps": 49.123760190990204, + "itl_mean_ms": 18.3165310353027, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.67446499870857, + "round_gap_p50_ms": 87.51248999760719, + "round_gap_p95_ms": 100.77195200210554, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3836656769999536, + "decode_tps": 55.77379644039523, + "prefill_tps": 36.49010281417927, + "itl_mean_ms": 17.92956663921359, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.22771599888802, + "round_gap_p50_ms": 86.77358999921125, + "round_gap_p95_ms": 99.8859840001387, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19076652500007185, + "decode_tps": 53.51315612831442, + "prefill_tps": 73.38813767244922, + "itl_mean_ms": 18.6869934862782, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.46596099910676, + "round_gap_p50_ms": 87.64228400104912, + "round_gap_p95_ms": 101.17966600228101, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.496441429413288, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.46596099910676, + "round_gap_p50_ms": 87.64228400104912, + "round_gap_p95_ms": 100.02229000019724, + "round_gap_mean_ms": 84.22486722322121, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 145.68675197346505 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30543.013170108, + "first": 30543.110326674, + "last": 30547.968758686, + "end": 30547.96920839, + "elapsed_s": 4.956038282001828, + "chunk_offsets_s": [ + 0.09715656600019429, + 0.44909214500148664, + 0.5109339660011756, + 0.569034610001836, + 0.6263118800015945, + 0.7039295230024436, + 0.789894388999528, + 0.8800957759995072, + 0.9732708020019345, + 1.062523799999326, + 1.1500197280001885, + 1.2408759400022973, + 1.3332170480025525, + 1.4317176380027377, + 1.5223360570016666, + 1.6132302050027647, + 1.7078062550026516, + 1.8006531380015076, + 1.8852476780011784, + 1.953932120999525, + 2.011531809999724, + 2.0789869940017525, + 2.153130938000686, + 2.245886092001456, + 2.336895899999945, + 2.422487452000496, + 2.4847828750025656, + 2.5440128810005262, + 2.6168730600002164, + 2.7050815139991755, + 2.7801169750018744, + 2.839549528001953, + 2.9054596810019575, + 2.9921676400008437, + 3.083737919001578, + 3.17713150200143, + 3.2786207190001733, + 3.3813346380011353, + 3.479072370999347, + 3.571473571999377, + 3.661321121999208, + 3.7510656170015864, + 3.843618768001761, + 3.935659550999844, + 4.024781195999822, + 4.115712472001178, + 4.197232425001857, + 4.255120205001731, + 4.326109804002044, + 4.425991944000998, + 4.509982642000978, + 4.575157417999435, + 4.647874813999806, + 4.7317115270016075, + 4.802776750999328, + 4.860857637999288, + 4.955588577999151 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.85219099991082, + "p95": 101.48921699874336, + "max": 351.93557900129235 + } + }, + { + "start": 30543.012845768, + "first": 30543.297840238, + "last": 30547.968555652, + "end": 30547.969339623, + "elapsed_s": 4.95649385499928, + "chunk_offsets_s": [ + 0.2849944699992193, + 0.4495060879999073, + 0.511046437000914, + 0.5701203440003155, + 0.6268117129984603, + 0.704028070998902, + 0.790718651998759, + 0.8803487810000661, + 0.9733514740000828, + 1.0629027680006402, + 1.1504152579982474, + 1.241561847000412, + 1.333493133999582, + 1.4322719010015135, + 1.5222278809997078, + 1.6134063549980056, + 1.7083798969979398, + 1.8009226030007994, + 1.885343310997996, + 1.9541666650002298, + 2.012270855000679, + 2.0796927429983043, + 2.153283586998441, + 2.2464643560015247, + 2.3369265840010485, + 2.4226672170007078, + 2.4851399949984625, + 2.5447461959993234, + 2.617253176998929, + 2.705486058999668, + 2.780308221001178, + 2.8397598400006245, + 2.9061049310003, + 2.9924485699993966, + 3.084230830998422, + 3.177980962998845, + 3.2787529150009505, + 3.3817308350007806, + 3.479176313998323, + 3.5718923259992152, + 3.6617790710006375, + 3.751475056000345, + 3.8441495209990535, + 3.9350531109994336, + 4.025163948001136, + 4.116119129001163, + 4.197333669999352, + 4.255396315998951, + 4.326786675999756, + 4.426480097998137, + 4.510180231998675, + 4.575387937999039, + 4.648331887998211, + 4.732095613999263, + 4.802987276998465, + 4.860746828999254, + 4.955709884001408 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.87268599917297, + "p95": 100.77195200210554, + "max": 164.51161800068803 + } + }, + { + "start": 30543.013442534, + "first": 30543.397108211, + "last": 30547.969147704, + "end": 30547.969504215, + "elapsed_s": 4.956061681001302, + "chunk_offsets_s": [ + 0.3836656769999536, + 0.4483291350006766, + 0.5097594770013529, + 0.5688917480001692, + 0.6253645270007837, + 0.7041467210001429, + 0.7903652640015935, + 0.8803692250003223, + 0.9731553230012651, + 1.0624085620002006, + 1.1503248369990615, + 1.2404864359996282, + 1.3327141519985162, + 1.4313647839990153, + 1.522282930000074, + 1.6131900680011313, + 1.7079920649994165, + 1.800696351001534, + 1.8849313680002524, + 1.9538043669999752, + 2.01142112999878, + 2.0788885440015292, + 2.1525702149992867, + 2.246168858000601, + 2.3367791210002906, + 2.4222717359989474, + 2.484688576998451, + 2.5440411119998316, + 2.6170455920000677, + 2.705005320000055, + 2.779586985001515, + 2.8390902999999525, + 2.9053833239995583, + 2.9921569139987696, + 3.0837630720016023, + 3.177629131001595, + 3.2779317620006623, + 3.381628972998442, + 3.478746060998674, + 3.571513431998028, + 3.6614901180000743, + 3.7511387829981686, + 3.8432304989983095, + 3.934974636998959, + 4.025259213998652, + 4.115834322998126, + 4.197243194998009, + 4.2551804590002575, + 4.326076463999925, + 4.425962448000064, + 4.509414840998943, + 4.575150315999053, + 4.6478513030015165, + 4.7317908920013, + 4.8028490780016, + 4.860778949001542, + 4.955705169999419 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.34493249903608, + "p95": 99.8859840001387, + "max": 103.69721099777962 + } + }, + { + "start": 30543.012721853, + "first": 30543.203488378, + "last": 30547.968671717, + "end": 30547.968773435, + "elapsed_s": 4.956051582001237, + "chunk_offsets_s": [ + 0.19076652500007185, + 0.449151246000838, + 0.5105965830007335, + 0.5703808319995005, + 0.6263536810001824, + 0.7046853470019414, + 0.790575995000836, + 0.8811371299998427, + 0.9737794790016778, + 1.0626987069990719, + 1.150340991000121, + 1.2413829110009829, + 1.333535349000158, + 1.4322592150019773, + 1.5223056350005209, + 1.6136408940001274, + 1.7084407340007601, + 1.8011526320005942, + 1.8854104839992942, + 1.9541945240016503, + 2.0118811130014365, + 2.07949960300175, + 2.1538060169987148, + 2.2465219699988666, + 2.33712463700067, + 2.422725281001476, + 2.4856108970016066, + 2.544828496000264, + 2.617618576001405, + 2.7054324250020727, + 2.780698309001309, + 2.8400525850011036, + 2.9063537500005623, + 2.9924733719999494, + 3.084718936999707, + 3.178018411999801, + 3.279198078002082, + 3.382075244000589, + 3.479394289999618, + 3.571860250998725, + 3.66183714699946, + 3.7514605989999836, + 3.843882665001729, + 3.9358300960011547, + 4.025117485001829, + 4.116403351999907, + 4.1978681910004525, + 4.25530172100116, + 4.326496107998537, + 4.426518397998734, + 4.510603929000354, + 4.575928764999844, + 4.648376076002023, + 4.732602208001481, + 4.803169755999988, + 4.86075120500027, + 4.955949864001013 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.7280665008584, + "p95": 101.17966600228101, + "max": 258.38472100076615 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 210.38618530253092, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 4.944831497999985, + "end_to_end_tps": 207.08491288614647, + "timing_valid": true, + "steady_tps": 220.55938820316823, + "per_stream_tps": [ + 52.6, + 54.8, + 53.6, + 55.9 + ], + "ttft_s": [ + 0.1, + 0.29, + 0.19, + 0.38 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.09629183799916063, + "decode_tps": 52.607377490675994, + "prefill_tps": 145.391346669715, + "itl_mean_ms": 19.008740745102486, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.52356700017117, + "round_gap_p50_ms": 87.37481400021352, + "round_gap_p95_ms": 97.58098900056211, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.288038285998482, + "decode_tps": 54.77310966729569, + "prefill_tps": 48.604649730743716, + "itl_mean_ms": 18.25713394901672, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.50422900004196, + "round_gap_p50_ms": 87.62966400172445, + "round_gap_p95_ms": 97.55090799808386, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.19112227599907783, + "decode_tps": 53.648611431102324, + "prefill_tps": 73.25153453105357, + "itl_mean_ms": 18.639811419615953, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.71068399903015, + "round_gap_p50_ms": 87.3627739965741, + "round_gap_p95_ms": 97.82365499995649, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38378960699992604, + "decode_tps": 55.909635700038564, + "prefill_tps": 36.47831974773277, + "itl_mean_ms": 17.886004576476076, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.61863000210724, + "round_gap_p50_ms": 87.25107300051604, + "round_gap_p95_ms": 97.24743699916871, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 18.447922672552806, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.50422900004196, + "round_gap_p50_ms": 87.62966400172445, + "round_gap_p95_ms": 97.24743699916871, + "round_gap_mean_ms": 84.00393359823154, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 145.91327899093108 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30547.970814019, + "first": 30548.067105857, + "last": 30552.914334747, + "end": 30552.914420189, + "elapsed_s": 4.943606169999839, + "chunk_offsets_s": [ + 0.09629183799916063, + 0.4526587250002194, + 0.5179863569974259, + 0.5756903209985467, + 0.633123085997795, + 0.7104619769997953, + 0.7984264859987888, + 0.8870823029974417, + 0.9776816669982509, + 1.0650564809984644, + 1.151428073997522, + 1.2398467809980502, + 1.3335125779994996, + 1.4297014490002766, + 1.5223031069981516, + 1.611975124997116, + 1.7072164979981608, + 1.7990676939989498, + 1.8841787759993167, + 1.9533101959968917, + 2.0101677530001325, + 2.078662281997822, + 2.1526339079973695, + 2.2448343910000403, + 2.3373579580002115, + 2.4202672190003796, + 2.4845219349990657, + 2.542693439998402, + 2.6138714689986955, + 2.701008837997506, + 2.776935322999634, + 2.8351436789998843, + 2.9016606639997917, + 2.9906504469981883, + 3.081398997997894, + 3.175534844998765, + 3.273115833999327, + 3.3694215399991663, + 3.4698030429972277, + 3.566704741999274, + 3.655133145999571, + 3.745284851996985, + 3.8375243800001044, + 3.9277974459982943, + 4.017289969997364, + 4.108421220997116, + 4.190604976996838, + 4.248127094997471, + 4.320585416997346, + 4.416427268999541, + 4.503408655000385, + 4.567115529000148, + 4.639332599999761, + 4.722265841999615, + 4.793743537997216, + 4.8495981539999775, + 4.943520728000294 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.66966149960354, + "p95": 97.58098900056211, + "max": 356.3668870010588 + } + }, + { + "start": 30547.970984176, + "first": 30548.259022462, + "last": 30552.914591619, + "end": 30552.914865511, + "elapsed_s": 4.943881334998878, + "chunk_offsets_s": [ + 0.288038285998482, + 0.4527856149979925, + 0.5176000249994104, + 0.5753413049969822, + 0.6327360949981085, + 0.709849224997015, + 0.7983617299978505, + 0.8869972669999697, + 0.9776596079973388, + 1.0652892719990632, + 1.1514965729984397, + 1.239955564000411, + 1.3336130789975869, + 1.4302701429987792, + 1.5219679179972445, + 1.6116691879979044, + 1.7072024640001473, + 1.7985245819982083, + 1.8843381029982993, + 1.9525673699972685, + 2.010137550998479, + 2.0788272529971437, + 2.152383450000343, + 2.2445652009992045, + 2.3373371309971844, + 2.4196069600002375, + 2.4849196399991342, + 2.5426296160003403, + 2.6138435670000035, + 2.7002847279982234, + 2.7771372770002927, + 2.834682992997841, + 2.9016282230004435, + 2.9905989059989224, + 3.081555157998082, + 3.175541857999633, + 3.273092765997717, + 3.3695653039976605, + 3.46974064999813, + 3.566872428997158, + 3.6549865739980305, + 3.7445714339992264, + 3.8370756629992684, + 3.928396048999275, + 4.017504851999547, + 4.107654399998864, + 4.190577354998823, + 4.24822967900036, + 4.320681248998881, + 4.416723553997144, + 4.503438114999881, + 4.56723233899902, + 4.6388193589991715, + 4.722336036000343, + 4.792881650999334, + 4.849238421997143, + 4.943607442997745 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.87190450129856, + "p95": 97.55090799808386, + "max": 164.7473289995105 + } + }, + { + "start": 30547.970735607, + "first": 30548.161857883, + "last": 30552.915009795, + "end": 30552.915048031, + "elapsed_s": 4.944312424002419, + "chunk_offsets_s": [ + 0.19112227599907783, + 0.4532069790002424, + 0.5183316740003647, + 0.5756523300005938, + 0.6328751770015515, + 0.7108983960024489, + 0.798261169999023, + 0.8875949899993429, + 0.9777063490000728, + 1.065807760001917, + 1.1518014120010776, + 1.2399853870010702, + 1.3334018349996768, + 1.4303254400001606, + 1.5224640349988476, + 1.612088739999308, + 1.7072097080017556, + 1.798951692002447, + 1.8848315290015307, + 1.9532830790012667, + 2.01017422500081, + 2.07924724500117, + 2.1523543110015453, + 2.2450649950005754, + 2.33777740000005, + 2.420126741999411, + 2.484688702999847, + 2.5424843190012325, + 2.6140361960024165, + 2.700731139000709, + 2.7771467929997016, + 2.8351005959993927, + 2.901924734000204, + 2.990623007000977, + 3.0816129240010923, + 3.175546618000226, + 3.2733702730001824, + 3.369633165999403, + 3.470224667998991, + 3.566850871000497, + 3.6551369600019825, + 3.7451155160015333, + 3.8375339430021995, + 3.927984893001849, + 4.01769731999957, + 4.108237968001049, + 4.1910886020014, + 4.2486389160003455, + 4.320862706001208, + 4.416758408999158, + 4.50325676400098, + 4.567334932999074, + 4.639361181001732, + 4.722168541000428, + 4.793635145000735, + 4.849638554998819, + 4.944274188001145 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.7320924992091, + "p95": 97.82365499995649, + "max": 262.0847030011646 + } + }, + { + "start": 30547.970612159, + "first": 30548.354401766, + "last": 30552.915332933, + "end": 30552.915443657, + "elapsed_s": 4.944831497999985, + "chunk_offsets_s": [ + 0.38378960699992604, + 0.4532673120011168, + 0.5179166950001672, + 0.5755727120013034, + 0.6328646900001331, + 0.7104690220003249, + 0.7982024140001158, + 0.8871948920022987, + 0.9779641130007803, + 1.0656289649996324, + 1.1517283390021475, + 1.2402654090001306, + 1.3339036899997154, + 1.4301415949994407, + 1.5228094939993753, + 1.6118575500004226, + 1.7071925199998077, + 1.7984808979999798, + 1.8848571500020626, + 1.9531563790005748, + 2.0104551710028318, + 2.0789382040020428, + 2.1522628090024227, + 2.2451417800002673, + 2.3377604100023746, + 2.4198112209996907, + 2.4851112780015683, + 2.5423803230005433, + 2.6142545450020407, + 2.7007161430010456, + 2.777364707999368, + 2.835107244001847, + 2.9019435200025328, + 2.991065778001939, + 3.0818796510029642, + 3.1756044710018614, + 3.273411712001689, + 3.369898655000725, + 3.470088198002486, + 3.5673356350016547, + 3.655724280000868, + 3.745429590002459, + 3.8375738180002372, + 3.92865170100049, + 4.017647771001066, + 4.10785200600003, + 4.1910030620019825, + 4.248645709001721, + 4.320897838999372, + 4.416487382000923, + 4.503738455001439, + 4.56754193800225, + 4.639249969000957, + 4.722800981002365, + 4.793473587000335, + 4.849934124002175, + 4.9447207740013255 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 87.45796249968407, + "p95": 97.24743699916871, + "max": 100.1895430017612 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "code", + "temperature": 0.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 241.7, + "per_stream_tps": 50.1, + "ttft_s_max": 0.49, + "request": { + "ttft_s": 0.288, + "decode_tps": 50.131, + "prefill_tps": 48.553, + "itl_mean_ms": 19.948, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.346, + "round_gap_p50_ms": 93.517, + "round_gap_p95_ms": 107.633, + "tokens_per_chunk": 4.491 + }, + "pooled": { + "aggregate_prefill_tps": 145.916, + "itl_mean_ms": 19.943, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.269, + "round_gap_p50_ms": 93.646, + "round_gap_p95_ms": 107.225, + "tokens_per_chunk": 4.491 + }, + "timing_valid": true, + "end_to_end_tps": 238.23957560420044, + "makespan_s": 5.3727429489990755, + "steady_tps": 256.4, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 241.8015008334509, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.3703899739994085, + "end_to_end_tps": 238.34395755188802, + "timing_valid": true, + "steady_tps": 256.41686568659856, + "per_stream_tps": [ + 51.1, + 52.1, + 50.1, + 49.2, + 48.4 + ], + "ttft_s": [ + 0.38, + 0.47, + 0.28, + 0.19, + 0.1 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.3757163820009737, + "decode_tps": 51.06909191578332, + "prefill_tps": 37.2621495113932, + "itl_mean_ms": 19.58131547843211, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.86210300141829, + "round_gap_p50_ms": 93.64564800125663, + "round_gap_p95_ms": 107.47200500190957, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.47196200200050953, + "decode_tps": 52.070639094330154, + "prefill_tps": 29.663404978913718, + "itl_mean_ms": 19.204680745101275, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.4471830019611, + "round_gap_p50_ms": 93.34643999682157, + "round_gap_p95_ms": 106.97869700015872, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28287855999951717, + "decode_tps": 50.13105740712364, + "prefill_tps": 49.49120216118145, + "itl_mean_ms": 19.94771408627618, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.89367300315644, + "round_gap_p50_ms": 93.68676799931563, + "round_gap_p95_ms": 107.79188899687142, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.189869293000811, + "decode_tps": 49.2379890771799, + "prefill_tps": 73.73493511633922, + "itl_mean_ms": 20.30952154509221, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.82211500048288, + "round_gap_p50_ms": 93.61646799879964, + "round_gap_p95_ms": 107.63312999915797, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09718869299831567, + "decode_tps": 48.37052839046334, + "prefill_tps": 144.04967870328935, + "itl_mean_ms": 20.673745631382403, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.59379899757914, + "round_gap_p50_ms": 93.666189997748, + "round_gap_p95_ms": 108.10698199929902, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.943395497256834, + "itl_p50_ms": 0.0, + "itl_p95_ms": 100.65851799663506, + "round_gap_p50_ms": 93.64564800125663, + "round_gap_p95_ms": 107.47200500190957, + "round_gap_mean_ms": 90.81367592500881, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 148.27596234155902 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30552.917413541, + "first": 30553.293129923, + "last": 30558.28636537, + "end": 30558.286507844, + "elapsed_s": 5.369094303001475, + "chunk_offsets_s": [ + 0.3757163820009737, + 0.5418860710014997, + 0.6110503580021032, + 0.670477089999622, + 0.7318043849991227, + 0.8167360059997009, + 0.9073691550001968, + 1.0012574540014612, + 1.1026253640011419, + 1.197443271001248, + 1.291359453000041, + 1.3913813710023533, + 1.4987551890008035, + 1.601577276000171, + 1.69721556100194, + 1.7936796550020517, + 1.8946453409989772, + 1.9942401990010694, + 2.087885847002326, + 2.161953089002054, + 2.2220939170001657, + 2.296022311998968, + 2.374811851001141, + 2.475939512001787, + 2.577704650000669, + 2.6654912820013124, + 2.7301376450013777, + 2.789241618000233, + 2.867462885002169, + 2.9594191319993115, + 3.0431970230019942, + 3.104336549000436, + 3.170771422999678, + 3.263317035001819, + 3.362941174000298, + 3.4647006500017596, + 3.5773888740004622, + 3.683783938002307, + 3.784442455998942, + 3.8853045590003603, + 3.981436873000348, + 4.080222930999298, + 4.178362497001217, + 4.274858041000698, + 4.371138788999815, + 4.472826753000845, + 4.5656818019997445, + 4.626840158001869, + 4.702693555002043, + 4.802032482999493, + 4.892735837001965, + 4.962625922002189, + 5.03958497900021, + 5.129022813001939, + 5.203028441999777, + 5.261479823999252, + 5.368951829001162 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.76697350126051, + "p95": 107.47200500190957, + "max": 166.16968900052598 + } + }, + { + "start": 30552.917090772, + "first": 30553.389052774, + "last": 30558.286246364, + "end": 30558.287350044, + "elapsed_s": 5.370259272000112, + "chunk_offsets_s": [ + 0.47196200200050953, + 0.5422671669984993, + 0.6107690720018581, + 0.6713334999985818, + 0.7323057840003457, + 0.8168324889993528, + 0.9072128650004743, + 1.0012618520013348, + 1.1019726599988644, + 1.197576956001285, + 1.291599538999435, + 1.3912377289998403, + 1.4996521690009104, + 1.6019946839987824, + 1.6972064760011563, + 1.7943716890003998, + 1.8956437179986096, + 1.9941611959984584, + 2.0876705049995508, + 2.162364683001215, + 2.221963350999431, + 2.2961471189992153, + 2.3748988340012147, + 2.476303318999271, + 2.578076942001644, + 2.666180633001204, + 2.730380184999376, + 2.789762847001839, + 2.867703094998433, + 2.959381728000153, + 3.0432178100018064, + 3.104285422999965, + 3.171305816998938, + 3.263109922001604, + 3.3627952419992653, + 3.465110081000603, + 3.577592014000402, + 3.6840011029999005, + 3.7844482860018616, + 3.8853086609997263, + 3.981704797999555, + 4.080397459001688, + 4.178546535000351, + 4.274742415000219, + 4.370944977999898, + 4.472376893001638, + 4.565723332998459, + 4.627474302000337, + 4.702793288000976, + 4.802111395998509, + 4.8932791109982645, + 4.96208132099855, + 5.039963919000002, + 5.129390877998958, + 5.203508943999623, + 5.262176895001176, + 5.369155592001334 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.42787449895695, + "p95": 106.97869700015872, + "max": 112.48193299979903 + } + }, + { + "start": 30552.917523236, + "first": 30553.200401796, + "last": 30558.287068888, + "end": 30558.287234411, + "elapsed_s": 5.3697111750007025, + "chunk_offsets_s": [ + 0.28287855999951717, + 0.5416520700018737, + 0.6107523890023003, + 0.6708134399996197, + 0.7313550120015861, + 0.8164665170006629, + 0.9069669470009103, + 1.0008776479990047, + 1.1017713210021611, + 1.1970657150013722, + 1.290976062999107, + 1.3910639980022097, + 1.4988558869990811, + 1.6012154210002336, + 1.6969151750017772, + 1.793743166999775, + 1.8946474500007753, + 1.9936043950001476, + 2.0872911629994633, + 2.1618857180001214, + 2.2215848439991532, + 2.295666658999835, + 2.3743615400017006, + 2.475554775999626, + 2.5781369550022646, + 2.6651320300006773, + 2.7291347620011948, + 2.788890738000191, + 2.8669780919990444, + 2.9588650920013606, + 3.0436407630004396, + 3.1041815630014753, + 3.170727247001196, + 3.2629163790006714, + 3.3627738499999396, + 3.4643217010016087, + 3.5772147540010337, + 3.683624838999094, + 3.7838821830009692, + 3.88508745800209, + 3.981027129000722, + 4.0793711289989005, + 4.178035323002405, + 4.274351895001018, + 4.370728755002347, + 4.472109215999808, + 4.564492182998947, + 4.6268129689997295, + 4.7024112489998515, + 4.801625244999741, + 4.892885149001813, + 4.961552144999587, + 5.039083618001314, + 5.129020512002171, + 5.202992794998863, + 5.262066193001374, + 5.3695456519999425 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.79855799852521, + "p95": 107.79188899687142, + "max": 258.7735100023565 + } + }, + { + "start": 30552.917650772, + "first": 30553.107520065, + "last": 30558.286448059, + "end": 30558.286780841, + "elapsed_s": 5.369130069000676, + "chunk_offsets_s": [ + 0.189869293000811, + 0.5418821329985803, + 0.6104265819994907, + 0.670343622998189, + 0.7316127349986346, + 0.816734993000864, + 0.90679974899831, + 1.0008512909989804, + 1.101917012998456, + 1.1970599279993621, + 1.290696003001358, + 1.3910277739996673, + 1.4986609039988252, + 1.6015911290014628, + 1.6969123929993657, + 1.793677244000719, + 1.8949624509987189, + 1.9938538380010868, + 2.0874703059998865, + 2.1615381419978803, + 2.222149321998586, + 2.2960298660000262, + 2.374516019001021, + 2.4759142059992882, + 2.577698353001324, + 2.665076623998175, + 2.7292685930005973, + 2.7887158000012278, + 2.866744634000497, + 2.95857215399883, + 3.042448208998394, + 3.1042423879989656, + 3.170682789001148, + 3.2626953049984877, + 3.3625146339982166, + 3.4643047969984764, + 3.576627448997897, + 3.683282688998588, + 3.783710769999743, + 3.884532885000226, + 3.980778179000481, + 4.079536846998963, + 4.17773618800129, + 4.274360986000829, + 4.370118277998699, + 4.472297938998963, + 4.564972054999089, + 4.627315009998711, + 4.702115681000578, + 4.801344921001146, + 4.892668706997938, + 4.962120560998301, + 5.039273491998756, + 5.1284333980001975, + 5.202617672999622, + 5.261749827997846, + 5.368797286999325 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.6262715003977, + "p95": 107.63312999915797, + "max": 352.01283999776933 + } + }, + { + "start": 30552.91696007, + "first": 30553.014148763, + "last": 30558.285953899, + "end": 30558.286023385, + "elapsed_s": 5.3690633149999485, + "chunk_offsets_s": [ + 0.09718869299831567, + 0.5420373449996987, + 0.6109586319980735, + 0.6711760369980766, + 0.7320715749992814, + 0.8171136300006765, + 0.9072792859988112, + 1.0015887580011622, + 1.1021825569987413, + 1.1978258560011454, + 1.2914920459988934, + 1.391565341000387, + 1.499672322999686, + 1.601708599999256, + 1.6974299369976507, + 1.7940339880005922, + 1.8953334739999264, + 1.9942303719981282, + 2.0879087840003194, + 2.1623392010005773, + 2.222237863999908, + 2.2963246119979885, + 2.374980559998221, + 2.4763185219999286, + 2.578463081998052, + 2.6661368169989146, + 2.730297789999895, + 2.7893466639980034, + 2.867622820998804, + 2.95937790699827, + 3.0430897969999933, + 3.1042799959977856, + 3.171486790997733, + 3.263437483998132, + 3.363279770997906, + 3.464945130999695, + 3.5774989609999466, + 3.684040146999905, + 3.7843404359991837, + 3.885315362000256, + 3.9815555479981413, + 4.079988833000243, + 4.178652818998671, + 4.275079655999434, + 4.37085895600103, + 4.473116110999399, + 4.565349968997907, + 4.627105034000124, + 4.702489079998486, + 4.801737099998718, + 4.893129587999283, + 4.962039560999983, + 5.039695629999187, + 5.129229982998368, + 5.20340502199906, + 5.262073809000867, + 5.3689938290008286 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.6723009999696, + "p95": 108.10698199929902, + "max": 444.848652001383 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 240.8069371792368, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.393478245001461, + "end_to_end_tps": 237.32366051281872, + "timing_valid": true, + "steady_tps": 256.0831948357464, + "per_stream_tps": [ + 49.2, + 50.1, + 52.0, + 48.2, + 51.0 + ], + "ttft_s": [ + 0.21, + 0.3, + 0.49, + 0.1, + 0.39 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.20736979600042105, + "decode_tps": 49.187215321721176, + "prefill_tps": 67.51224271818049, + "itl_mean_ms": 20.33048615294141, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.35698900153511, + "round_gap_p50_ms": 94.56506900096429, + "round_gap_p95_ms": 107.5081040035002, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3003836530006083, + "decode_tps": 50.073132536609165, + "prefill_tps": 46.60706353408535, + "itl_mean_ms": 19.97078970980867, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.92444999786676, + "round_gap_p50_ms": 95.0920799987216, + "round_gap_p95_ms": 107.85764499814832, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.4882378570000583, + "decode_tps": 52.00506107463322, + "prefill_tps": 28.67454827452745, + "itl_mean_ms": 19.228897713722237, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.22347600149806, + "round_gap_p50_ms": 92.61350899760146, + "round_gap_p95_ms": 105.65242499797023, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09811958499994944, + "decode_tps": 48.16800104767143, + "prefill_tps": 142.68303315803072, + "itl_mean_ms": 20.76067053333413, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.31051099920296, + "round_gap_p50_ms": 93.517035998957, + "round_gap_p95_ms": 107.31216600106563, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.3938023480004631, + "decode_tps": 51.00753952643597, + "prefill_tps": 35.55082916870657, + "itl_mean_ms": 19.604944862743757, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.482944002375, + "round_gap_p50_ms": 95.05705199990189, + "round_gap_p95_ms": 107.81525900165434, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.97915779451004, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.26920699945185, + "round_gap_p50_ms": 95.05705199990189, + "round_gap_p95_ms": 106.40393499852507, + "round_gap_mean_ms": 90.97652210000108, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 143.21340168447966 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30558.289273663, + "first": 30558.496643459, + "last": 30563.680917428, + "end": 30563.6810609, + "elapsed_s": 5.391787237000244, + "chunk_offsets_s": [ + 0.20736979600042105, + 0.5550750990005326, + 0.6224699030026386, + 0.6851628130025347, + 0.7456940510019194, + 0.8309437310017529, + 0.920910979002656, + 1.0162624050026352, + 1.12015041199993, + 1.2194067250020453, + 1.3139717940030096, + 1.4121562410000479, + 1.5141372560028685, + 1.6134158180029772, + 1.7088542270030302, + 1.8058166150003672, + 1.9083831510033633, + 2.0078936990030343, + 2.104136797002866, + 2.181524839001213, + 2.2428426090009452, + 2.31623951200163, + 2.3961153640011617, + 2.492900768000254, + 2.591421748002176, + 2.6799043890023313, + 2.74568986000304, + 2.8063421470033063, + 2.8832713310002873, + 2.9747924490002333, + 3.057370668000658, + 3.1238505280016398, + 3.1950913700020465, + 3.287537310003245, + 3.384921632001351, + 3.484477110003354, + 3.5925627579999855, + 3.7000708620034857, + 3.8007402540024486, + 3.903394926001056, + 4.001803495000786, + 4.100489340002241, + 4.205805595000129, + 4.301767852000921, + 4.3984770070019295, + 4.496545281002909, + 4.586335017000238, + 4.647504933000164, + 4.723569917001441, + 4.8229269060029765, + 4.9126455370023905, + 4.98006741500285, + 5.057306972001243, + 5.1498791110025195, + 5.226982501000748, + 5.28591206600322, + 5.391643765000481 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 94.95824750047177, + "p95": 107.5081040035002, + "max": 347.7053030001116 + } + }, + { + "start": 30558.289042284, + "first": 30558.589425937, + "last": 30563.681977313, + "end": 30563.68233292, + "elapsed_s": 5.393290636002348, + "chunk_offsets_s": [ + 0.3003836530006083, + 0.5555784960015444, + 0.6225971769999887, + 0.6855468319990905, + 0.7463179350015707, + 0.830847844001255, + 0.9208914600021672, + 1.0161026290006703, + 1.1202575620009156, + 1.219380557002296, + 1.3144726370010176, + 1.4122922850001487, + 1.5144200390022888, + 1.6133444890001556, + 1.708888934001152, + 1.806136188999517, + 1.9077318260024185, + 2.008321283999976, + 2.103865824999957, + 2.1821630190024734, + 2.242785572001594, + 2.3163344070017047, + 2.3965795959993557, + 2.4938728910019563, + 2.5920262610015925, + 2.6799786650008173, + 2.745829634000984, + 2.8066499900014605, + 2.8834644779999508, + 2.9746972600005392, + 3.0579945140016207, + 3.1243066730021383, + 3.19517619600083, + 3.2886082810000516, + 3.3853514600014023, + 3.4840088180026214, + 3.592044271001214, + 3.6999019159993622, + 3.8004640100007236, + 3.9034849060008128, + 4.002218146000814, + 4.1004427850020875, + 4.205876436000835, + 4.301530045002437, + 4.398043805002089, + 4.496635068000614, + 4.5866802650016325, + 4.647663829000521, + 4.724419767000654, + 4.822697944000538, + 4.912370195001131, + 4.980680633001612, + 5.0573366360004, + 5.149919804000092, + 5.227761089001433, + 5.28663846299969, + 5.392935029001819 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 95.15162449861236, + "p95": 107.85764499814832, + "max": 255.1948430009361 + } + }, + { + "start": 30558.28939789, + "first": 30558.777635747, + "last": 30563.681004664, + "end": 30563.681556014, + "elapsed_s": 5.392158123999252, + "chunk_offsets_s": [ + 0.4882378570000583, + 0.5548925719995168, + 0.6223858910016133, + 0.6846990660014853, + 0.7454440540022915, + 0.8306932359992061, + 0.920884240000305, + 1.0162514700023166, + 1.1198279810014355, + 1.2200109580007847, + 1.3136196350023965, + 1.411846595001407, + 1.5139430119997996, + 1.613207186001091, + 1.7089835200022208, + 1.8059664310021617, + 1.9081629239990434, + 2.007917325001472, + 2.1041963630013925, + 2.1814693109990912, + 2.2428732049993414, + 2.316479289002018, + 2.3962948730004427, + 2.49356865700247, + 2.59137371199904, + 2.6795148509991122, + 2.7450944550000713, + 2.806179881001299, + 2.8831753650010796, + 2.9746167920020525, + 3.0575725920025434, + 3.1233432910012198, + 3.195397199000581, + 3.2877545850024035, + 3.3847486400009075, + 3.4837785540003097, + 3.5923027669996372, + 3.6996340370023972, + 3.800314378000621, + 3.9028174400009448, + 4.001274543999898, + 4.100498020001396, + 4.204843694002193, + 4.301934949999122, + 4.397886402999575, + 4.496679587002291, + 4.5864139500008605, + 4.647344793000229, + 4.723573589999432, + 4.822519443001511, + 4.912657975000911, + 4.979910181002197, + 5.0570870010014914, + 5.149700509999093, + 5.226940871001716, + 5.285954349001258, + 5.391606773999229 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.11109299960663, + "p95": 105.65242499797023, + "max": 108.52421299932757 + } + }, + { + "start": 30558.289159764, + "first": 30558.387279349, + "last": 30563.681250335, + "end": 30563.681663735, + "elapsed_s": 5.392503970997495, + "chunk_offsets_s": [ + 0.09811958499994944, + 0.5553156189998845, + 0.622400389998802, + 0.6846964119977201, + 0.7459620349982288, + 0.8306114029983291, + 0.9209643299982417, + 1.016573688997596, + 1.120336887997837, + 1.2199164049998217, + 1.3134334409987787, + 1.411798135999561, + 1.5139048149976588, + 1.6129733929992653, + 1.7086674899983336, + 1.8057099309990008, + 1.9080990490001568, + 2.0078984139981912, + 2.104302051997365, + 2.1819632699989597, + 2.2433226319990354, + 2.315880318998097, + 2.3957162330007122, + 2.494090766998852, + 2.5917315269980463, + 2.679907027999434, + 2.7456733920007537, + 2.8060668389989587, + 2.883121814000333, + 2.974442283000826, + 3.0579784980000113, + 3.1241421140002785, + 3.1951360149978427, + 3.288164601999597, + 3.3851129579998087, + 3.4844234689990117, + 3.5917356350000773, + 3.699710018998303, + 3.8004517410008702, + 3.9031910249977955, + 4.001250966997759, + 4.1004097719996935, + 4.204800577997958, + 4.301302103998751, + 4.397762077998777, + 4.496392581000691, + 4.586308187997929, + 4.647280450000835, + 4.724155988998973, + 4.822628529000212, + 4.912461670999619, + 4.980223374997877, + 5.05736461899869, + 5.149562271999457, + 5.227469184999791, + 5.286292762997618, + 5.392090571000153 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 94.56319749915565, + "p95": 107.31216600106563, + "max": 457.196033999935 + } + }, + { + "start": 30558.288854675, + "first": 30558.682657023, + "last": 30563.681917963, + "end": 30563.682112582, + "elapsed_s": 5.39325790699877, + "chunk_offsets_s": [ + 0.3938023480004631, + 0.5553812450016267, + 0.6231234229999245, + 0.6858276730017678, + 0.7460348720014736, + 0.8313075510013732, + 0.9209834760004014, + 1.0160405280003033, + 1.1202807429981476, + 1.2197636870005226, + 1.3148231160012074, + 1.4122892550003598, + 1.5144517970002198, + 1.6138844119996065, + 1.7093535249987326, + 1.806775804998324, + 1.90866607299904, + 2.0084077710016572, + 2.1038570839991735, + 2.1820975900009216, + 2.243223088000377, + 2.3170869640016463, + 2.3962387579995266, + 2.4932699099990714, + 2.592432342000393, + 2.6805452209991927, + 2.7458028010005364, + 2.806517051998526, + 2.8836011060011515, + 2.9755592010005785, + 3.0577345159981633, + 3.1239250590006122, + 3.195561403001193, + 3.288371565999114, + 3.384963358999812, + 3.484526800999447, + 3.5924283169988485, + 3.700243576000503, + 3.800929681998241, + 3.9033998459999566, + 4.002268385000207, + 4.1004860879984335, + 4.2059013399994, + 4.302615117998357, + 4.39863382600015, + 4.497123924000334, + 4.586792710000736, + 4.647665775999485, + 4.72416512700147, + 4.823434334000922, + 4.912859454001591, + 4.980375759998424, + 5.057772355998168, + 5.150359232000483, + 5.227708035999967, + 5.286659353001596, + 5.393063288000121 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 95.05824050029332, + "p95": 107.81525900165434, + "max": 161.57889700116357 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 241.7454340237839, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 5.3727429489990755, + "end_to_end_tps": 238.23957560420044, + "timing_valid": true, + "steady_tps": 256.705204765509, + "per_stream_tps": [ + 49.2, + 48.4, + 52.1, + 50.2, + 51.1 + ], + "ttft_s": [ + 0.19, + 0.1, + 0.48, + 0.29, + 0.38 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 14, + "ttft_s": 0.19371499200133258, + "decode_tps": 49.24897546010529, + "prefill_tps": 72.27112292838798, + "itl_mean_ms": 20.304990929406475, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.7156569972285, + "round_gap_p50_ms": 93.1067460005579, + "round_gap_p95_ms": 108.07434799789917, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.09742605900100898, + "decode_tps": 48.35470828594897, + "prefill_tps": 143.6987202762149, + "itl_mean_ms": 20.680509415679435, + "itl_p50_ms": 0.0, + "itl_p95_ms": 99.34640700157615, + "round_gap_p50_ms": 93.09267199932947, + "round_gap_p95_ms": 107.47548200015444, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.47862166200138745, + "decode_tps": 52.132305575745484, + "prefill_tps": 29.250661036648644, + "itl_mean_ms": 19.181963831372332, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.57234500304912, + "round_gap_p50_ms": 91.96488900124677, + "round_gap_p95_ms": 106.3220830001228, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.28834356399966055, + "decode_tps": 50.166164086897886, + "prefill_tps": 48.553190526619424, + "itl_mean_ms": 19.933754517642583, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.56614800082752, + "round_gap_p50_ms": 93.32255200206419, + "round_gap_p95_ms": 107.76304200044251, + "tokens_per_chunk": 4.491228070175438 + }, + { + "prompt_tokens": 14, + "ttft_s": 0.38435483099965495, + "decode_tps": 51.12946843450504, + "prefill_tps": 36.424675510355605, + "itl_mean_ms": 19.55819277254883, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.85866699914914, + "round_gap_p50_ms": 93.45543200106476, + "round_gap_p95_ms": 107.72782300045947, + "tokens_per_chunk": 4.491228070175438 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 19.93188229332993, + "itl_p50_ms": 0.0, + "itl_p95_ms": 98.7156569972285, + "round_gap_p50_ms": 93.32255200206419, + "round_gap_p95_ms": 107.22530799830565, + "round_gap_mean_ms": 90.76124972855594, + "tokens_per_chunk": 4.491228070175438, + "aggregate_prefill_tps": 145.9156750691102 + }, + "evidence": [ + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + }, + { + "prompt_tokens": 14, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d8f550c1ded0", + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18" + } + ], + "client_intervals": [ + { + "start": 30563.683884003, + "first": 30563.877598995, + "last": 30569.055371682, + "end": 30569.055596104, + "elapsed_s": 5.371712101001322, + "chunk_offsets_s": [ + 0.19371499200133258, + 0.5455536299996311, + 0.6134315980016254, + 0.6733740999989095, + 0.7390051299989864, + 0.826452117998997, + 0.9151564749990939, + 1.0091486330020416, + 1.1089083240003674, + 1.2045517389997258, + 1.2986198130020057, + 1.3967948040008196, + 1.4977771529993333, + 1.5951536870015843, + 1.690310429999954, + 1.7943339640005433, + 1.897903810000571, + 1.9962972829998762, + 2.0883159750010236, + 2.1632088139995176, + 2.2226748120010598, + 2.295837901001505, + 2.3752775249995466, + 2.4723328610016324, + 2.571048517998861, + 2.6591803880000953, + 2.728531037999346, + 2.7912867679988267, + 2.8693156049994286, + 2.961143283999263, + 3.0432199999995646, + 3.104702044998703, + 3.1743181150013697, + 3.2674248610019276, + 3.365244675998838, + 3.4640021459999844, + 3.570816735002154, + 3.676145188001101, + 3.785208198001783, + 3.887001440001768, + 3.982577922000928, + 4.079484025001875, + 4.177722265001648, + 4.274888741001632, + 4.372985264999443, + 4.471091400999285, + 4.561268923000171, + 4.621583030999318, + 4.697396581999783, + 4.802589860999433, + 4.8938463410013355, + 4.96314011800132, + 5.041829230001895, + 5.1298158099998545, + 5.205380652001622, + 5.263413331002084, + 5.3714876789999835 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.5494520017528, + "p95": 108.07434799789917, + "max": 351.8386379982985 + } + }, + { + "start": 30563.684005305, + "first": 30563.781431364, + "last": 30569.054961265, + "end": 30569.055040704, + "elapsed_s": 5.3710353989990836, + "chunk_offsets_s": [ + 0.09742605900100898, + 0.5455266019998817, + 0.6135691689996747, + 0.6736449370000628, + 0.7383134289993905, + 0.8261978720001935, + 0.9147707769989211, + 1.0088346720003756, + 1.1087150349994772, + 1.203898774998379, + 1.2977130069994018, + 1.3970094260002952, + 1.4975205119990278, + 1.5949094850002439, + 1.690354466001736, + 1.7944894879983622, + 1.897435252001742, + 1.9954058539988182, + 2.0874125720001757, + 2.1627141110002412, + 2.2222602110014122, + 2.2952333409994026, + 2.3745429670016165, + 2.4716272719997505, + 2.5709736790013267, + 2.6588970049997442, + 2.7283534519992827, + 2.791232086998207, + 2.8689400190014567, + 2.9605455769997207, + 3.042384493001009, + 3.104083342001104, + 3.17377244599993, + 3.2668651179992594, + 3.3647749929987185, + 3.4641687250004907, + 3.5702305609993346, + 3.675749549998727, + 3.785781828999461, + 3.8868150650014286, + 3.982250582001143, + 4.078857137999876, + 4.177344060000905, + 4.274861824000254, + 4.372482235998177, + 4.470561237001675, + 4.560239636000915, + 4.621613366998645, + 4.697366423999483, + 4.802656462001323, + 4.893523239999922, + 4.962879260998307, + 5.04135019000023, + 5.129239983998559, + 5.20514371399986, + 5.26348047799911, + 5.3709559599992645 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.45345200017618, + "p95": 107.47548200015444, + "max": 448.1005429988727 + } + }, + { + "start": 30563.684898874, + "first": 30564.163520536, + "last": 30569.054921313, + "end": 30569.055786494, + "elapsed_s": 5.3708876199998485, + "chunk_offsets_s": [ + 0.47862166200138745, + 0.5450645000018994, + 0.6123106700033532, + 0.6724546750010632, + 0.7378661910006485, + 0.8255022750017815, + 0.9144800000030955, + 1.008627029001218, + 1.1079296000025352, + 1.2036753690008481, + 1.2971720870009449, + 1.3960077720003028, + 1.497208266002417, + 1.5937976590030303, + 1.6893959720000566, + 1.7936692069997662, + 1.896797419001814, + 1.9947307990005356, + 2.0866956880017824, + 2.162047863002954, + 2.2214871710020816, + 2.2944433060001757, + 2.373912172002747, + 2.471437476000574, + 2.5698250720015494, + 2.65852909500245, + 2.727876439999818, + 2.790710378001677, + 2.8694256220005627, + 2.9598836460027087, + 3.0417591860023094, + 3.103371787001379, + 3.1731674000002386, + 3.265915269003017, + 3.364030556000216, + 3.463178981000965, + 3.569501064001088, + 3.67463487400164, + 3.785268351002742, + 3.885868041001231, + 3.981788666002103, + 4.078244614000141, + 4.17681695900319, + 4.273723066002276, + 4.371634608000022, + 4.46975032300179, + 4.5598594390030485, + 4.62008737799988, + 4.696290199000941, + 4.801736240002356, + 4.893030846000329, + 4.9628154180027195, + 5.0405659770003695, + 5.128281593002612, + 5.204587636002543, + 5.262797131003026, + 5.370022439001332 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 92.35637900201255, + "p95": 106.3220830001228, + "max": 110.63347700110171 + } + }, + { + "start": 30563.684123445, + "first": 30563.972467009, + "last": 30569.055574411, + "end": 30569.056259454, + "elapsed_s": 5.3721360089984955, + "chunk_offsets_s": [ + 0.28834356399966055, + 0.5454829119989881, + 0.6135145640000701, + 0.6729439580012695, + 0.7385946840004181, + 0.8264517899988277, + 0.9153183859998535, + 1.0092587270009972, + 1.108933943000011, + 1.2043812829979288, + 1.297703834999993, + 1.3966051989991684, + 1.498195724998368, + 1.5946588309998333, + 1.6902755699993577, + 1.7940043499984313, + 1.8981915849981306, + 1.9957459120014391, + 2.0879731019995234, + 2.162469743998372, + 2.22284037099962, + 2.2956805339999846, + 2.3750879479994182, + 2.47203731099944, + 2.5705292579987145, + 2.658982223998464, + 2.728152384999703, + 2.7909637580014532, + 2.870019695001247, + 2.9614455019982415, + 3.042767505001393, + 3.104580489998625, + 3.1739929619980103, + 3.2673818439980096, + 3.3651453950005816, + 3.463711543001409, + 3.5710928269982105, + 3.6759733549988596, + 3.785473859999911, + 3.8871735430002445, + 3.9832142230006866, + 4.079700884998601, + 4.177423675999307, + 4.2747046270014835, + 4.37301373399896, + 4.471488230999967, + 4.560246353001276, + 4.621376708000753, + 4.696910153998033, + 4.802447949001362, + 4.8935605570004554, + 4.963370900000882, + 5.041855965999275, + 5.129650561000744, + 5.20524197299892, + 5.263687923998077, + 5.37145096599852 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.35571700103173, + "p95": 107.76304200044251, + "max": 257.1393479993276 + } + }, + { + "start": 30563.683791405, + "first": 30564.068146236, + "last": 30569.055485393, + "end": 30569.056534354, + "elapsed_s": 5.3727429489990755, + "chunk_offsets_s": [ + 0.38435483099965495, + 0.5462966459999734, + 0.6135671349984477, + 0.6736167369999748, + 0.7384251090006728, + 0.8266919950001466, + 0.9154882339971664, + 1.0096425349984202, + 1.109326413999952, + 1.2043339329975424, + 1.29843235899898, + 1.3974316479980189, + 1.4979791849982576, + 1.5953125429987267, + 1.6903103829972679, + 1.7945059809972008, + 1.8982713730001706, + 1.99603007299811, + 2.088118296000175, + 2.163403163998737, + 2.2232216819975292, + 2.2956280349972076, + 2.3755823579995194, + 2.472252159997879, + 2.570962038997095, + 2.6594997769971087, + 2.7288037719990825, + 2.7916340509982547, + 2.87057067799833, + 2.9616171779998695, + 3.043142765000084, + 3.1049440469978435, + 3.1738601139986713, + 3.267315545999736, + 3.3652977159981674, + 3.4641563829973165, + 3.5711140540006454, + 3.6763337519987545, + 3.785496836000675, + 3.8874517749973165, + 3.98262479999903, + 4.079658482998639, + 4.177996379999968, + 4.275312952999229, + 4.373156329998892, + 4.471375512999657, + 4.560677093999402, + 4.621322191000218, + 4.697318350998103, + 4.802560281998012, + 4.893979632997798, + 4.963672065998253, + 5.042028649000713, + 5.129655168999307, + 5.2055471700005, + 5.263966164999147, + 5.371693987999606 + ], + "chunk_gaps_ms": { + "count": 56, + "p50": 93.77692900125112, + "p95": 107.72782300045947, + "max": 161.94181500031846 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d8f550c1ded04d67a59b46a3daa2291b55159523d39744845378b11b3e828624", + "output_sha256": "38e4a0109d0ddd70d488184b786c38a08ee9bf5ebd2daedaeca65dd59f8cea18", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.6, + "per_stream_tps": 42.6, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.087, + "decode_tps": 42.561, + "prefill_tps": null, + "itl_mean_ms": 23.496, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.951, + "round_gap_p50_ms": 59.846, + "round_gap_p95_ms": 79.13, + "tokens_per_chunk": 2.56 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 23.496, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.951, + "round_gap_p50_ms": 59.846, + "round_gap_p95_ms": 79.13, + "tokens_per_chunk": 2.56 + }, + "timing_valid": true, + "end_to_end_tps": 42.1164970684955, + "makespan_s": 6.078378255999269, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.60049811088277, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.072872966000432, + "end_to_end_tps": 42.154677272724264, + "timing_valid": true, + "per_stream_tps": [ + 42.6 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08688690099734231, + "decode_tps": 42.60049811088277, + "prefill_tps": null, + "itl_mean_ms": 23.47390392941295, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.95114300015848, + "round_gap_p50_ms": 59.944540000287816, + "round_gap_p95_ms": 79.12956400105031, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.47390392941295, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.95114300015848, + "round_gap_p50_ms": 59.944540000287816, + "round_gap_p95_ms": 79.12956400105031, + "round_gap_mean_ms": 60.46308587879093, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30585.599999662, + "first": 30585.686886563, + "last": 30591.672732065, + "end": 30591.672872628, + "elapsed_s": 6.072872966000432, + "chunk_offsets_s": [ + 0.08688690099734231, + 0.14629199000046356, + 0.21295103000011295, + 0.28894325099827256, + 0.34428477499750443, + 0.39278700199793093, + 0.4398474079971493, + 0.49224367999704555, + 0.5496503270005633, + 0.6141479209982208, + 0.6814623859972926, + 0.752437323997583, + 0.8266204009996727, + 0.8893814089969965, + 0.9400269260004279, + 0.9968271799989452, + 1.0593822800001362, + 1.1208760409972456, + 1.1810956150002312, + 1.242108635000477, + 1.3015438740003447, + 1.3641795839976112, + 1.4367340879980475, + 1.514827779999905, + 1.5886993629974313, + 1.6475484330003383, + 1.7000720479991287, + 1.7518354569983785, + 1.8040896269994846, + 1.8515178889974777, + 1.91264860499723, + 1.9781540849980956, + 2.023723460999463, + 2.07084778799981, + 2.1226403019973077, + 2.173646485000063, + 2.219033209999907, + 2.2799979649971647, + 2.3381984699990426, + 2.396729290998337, + 2.4545857599987357, + 2.5028378749993863, + 2.562891186997149, + 2.623670294000476, + 2.6728980379994027, + 2.731805395000265, + 2.800106777998735, + 2.863152120997256, + 2.9275852069986286, + 2.9929263699996227, + 3.0537103489987203, + 3.101836434998404, + 3.1531115919970034, + 3.1990479809974204, + 3.2434120919970155, + 3.289310307998676, + 3.34846673499851, + 3.4061010099976556, + 3.4667055969985086, + 3.5312659539995366, + 3.5804252339985396, + 3.6411544999973557, + 3.7152217509974435, + 3.7849576259977766, + 3.866548068999691, + 3.942340224999498, + 4.0109621999981755, + 4.070853604000149, + 4.1530319589983264, + 4.224187395997433, + 4.272886302998813, + 4.317610203997901, + 4.380322946999513, + 4.454274089999672, + 4.5166494459990645, + 4.572294311998121, + 4.629023417997814, + 4.683603595000022, + 4.737469930998486, + 4.795613482998306, + 4.845338412000274, + 4.907121394000569, + 4.985326197998802, + 5.064455761999852, + 5.137844369997765, + 5.196373143997334, + 5.257301647998247, + 5.335130076000496, + 5.414359895999951, + 5.495581213999685, + 5.560673722997308, + 5.613288221997209, + 5.670640293999895, + 5.7258090269970126, + 5.783884910997585, + 5.843829450997873, + 5.896123305999936, + 5.944490955000219, + 6.004544667997834, + 6.072732402997644 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.944540000287816, + "p95": 79.12956400105031, + "max": 82.17835499817738 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.56112171479661, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.078378255999269, + "end_to_end_tps": 42.1164970684955, + "timing_valid": true, + "per_stream_tps": [ + 42.6 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08687853400260792, + "decode_tps": 42.56112171479661, + "prefill_tps": null, + "itl_mean_ms": 23.495621349010275, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.98637400203734, + "round_gap_p50_ms": 59.84574800095288, + "round_gap_p95_ms": 79.22316599797341, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.495621349010275, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.98637400203734, + "round_gap_p50_ms": 59.84574800095288, + "round_gap_p95_ms": 79.22316599797341, + "round_gap_mean_ms": 60.519024686844645, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30591.673729264, + "first": 30591.760607798, + "last": 30597.751991242, + "end": 30597.75210752, + "elapsed_s": 6.078378255999269, + "chunk_offsets_s": [ + 0.08687853400260792, + 0.14684080699953483, + 0.21424556000056327, + 0.2892319340026006, + 0.3457808230014052, + 0.3955900799992378, + 0.4428623990024789, + 0.4938660650004749, + 0.551109718002408, + 0.619200085002376, + 0.6837448510013928, + 0.7594560479992651, + 0.8311858670022048, + 0.8897021709999535, + 0.9405580679995182, + 0.9966953139992256, + 1.059115746000316, + 1.1190338600026735, + 1.1783609279991651, + 1.2413405420011259, + 1.299220247001358, + 1.3606862900014676, + 1.4339644360006787, + 1.5131190070023877, + 1.5865241659994354, + 1.643293518001883, + 1.695289364000928, + 1.7527006240015908, + 1.8012828589999117, + 1.848632773002464, + 1.9087956590010435, + 1.9740035590002663, + 2.019393458002014, + 2.066479204000643, + 2.117524716002663, + 2.1689780350025103, + 2.2156054440019943, + 2.2752310060022864, + 2.3332433180003136, + 2.390501813002629, + 2.4475281449995236, + 2.496541544001957, + 2.5547826530018938, + 2.619972881002468, + 2.6674692750020768, + 2.7286305730012828, + 2.7973840910017316, + 2.857685713999672, + 2.9209892860017135, + 2.9855226809995656, + 3.0461376669991296, + 3.0955315950013755, + 3.146145836002688, + 3.1936681420011155, + 3.2399790000017674, + 3.2858875849997276, + 3.346088881000469, + 3.405436589000601, + 3.4645573500019964, + 3.5293202770008065, + 3.5794013119993906, + 3.6407414290006272, + 3.715941799000575, + 3.7907664449994627, + 3.869302371000231, + 3.9413034499993955, + 4.009866995002085, + 4.071186749002663, + 4.153370560001349, + 4.224209345000418, + 4.271520392001548, + 4.316370898999594, + 4.379117896001844, + 4.454449084001681, + 4.5159300769992115, + 4.57181872000001, + 4.631664468000963, + 4.684496394002053, + 4.744596750002529, + 4.8034522970010585, + 4.848660794999887, + 4.909448521000741, + 4.989075453002442, + 5.068371708999621, + 5.143387078001979, + 5.2041276800009655, + 5.26480908600206, + 5.344032252000034, + 5.42429523300234, + 5.503438456002186, + 5.5697050010021485, + 5.6219590900000185, + 5.680403351001587, + 5.739487810002174, + 5.798826307000127, + 5.854317704001005, + 5.9045269430025655, + 5.951618065999355, + 6.010773320002045, + 6.078261978000228 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.84574800095288, + "p95": 79.22316599797341, + "max": 82.18381099868566 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 42.54355943583867, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 6.080568739998853, + "end_to_end_tps": 42.10132488364046, + "timing_valid": true, + "per_stream_tps": [ + 42.5 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.0865992830003961, + "decode_tps": 42.54355943583867, + "prefill_tps": null, + "itl_mean_ms": 23.505320505871936, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.7347200010845, + "round_gap_p50_ms": 59.68404300074326, + "round_gap_p95_ms": 78.86902099926374, + "tokens_per_chunk": 2.56 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 23.505320505871936, + "itl_p50_ms": 0.0, + "itl_p95_ms": 73.7347200010845, + "round_gap_p50_ms": 59.68404300074326, + "round_gap_p95_ms": 78.86902099926374, + "round_gap_mean_ms": 60.54400736360953, + "tokens_per_chunk": 2.56, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30597.753077197, + "first": 30597.83967648, + "last": 30603.833533209, + "end": 30603.833645937, + "elapsed_s": 6.080568739998853, + "chunk_offsets_s": [ + 0.0865992830003961, + 0.1461305419979908, + 0.21342155499951332, + 0.2899072529980913, + 0.3470522319985321, + 0.39510261899704346, + 0.44377187699865317, + 0.49610656699951505, + 0.555117463998613, + 0.6192847780002921, + 0.6891153199976543, + 0.7628500399987388, + 0.8321810929992353, + 0.8907136149973667, + 0.941346176998195, + 0.9986693619975995, + 1.0627642479994392, + 1.1257258639998327, + 1.1859132190002128, + 1.2489036379993195, + 1.307358498997928, + 1.3695469009981025, + 1.4423242209995806, + 1.5209934569975303, + 1.5947125819984649, + 1.6551381629979005, + 1.710629268996854, + 1.7636080960000982, + 1.8117691649968037, + 1.8580063869994774, + 1.9174091539971414, + 1.9818819389984128, + 2.0268842320001568, + 2.0731915429969376, + 2.123114144000283, + 2.1747861329968146, + 2.220451934997982, + 2.281039901998156, + 2.339181251998525, + 2.3960343209982966, + 2.4530609239991463, + 2.502207001998613, + 2.5631966139990254, + 2.6249998639978003, + 2.6779385409972747, + 2.7396675589989172, + 2.805167454000184, + 2.8650986409993493, + 2.925860654999269, + 2.988765340996906, + 3.0487932010000804, + 3.0980294319997483, + 3.149721576999582, + 3.1957055169987143, + 3.2405159560003085, + 3.286575471996912, + 3.3479175020002003, + 3.4043478649982717, + 3.463456942998164, + 3.5282749789985246, + 3.577989252000407, + 3.6402835509979923, + 3.720448303996818, + 3.7907785889983643, + 3.8678755039982207, + 3.9407628279986966, + 4.008650575997308, + 4.069658614000218, + 4.153252066997084, + 4.223374391996913, + 4.272183129000041, + 4.317376861999946, + 4.380595689999609, + 4.454653573997348, + 4.520280476997868, + 4.5784668139967835, + 4.636855908000143, + 4.694077508000191, + 4.751659419998759, + 4.80912700999761, + 4.8540471859996615, + 4.913032344000385, + 4.991901364999649, + 5.0702189759977045, + 5.1451697219999915, + 5.204526940997312, + 5.26515611799914, + 5.344289572996786, + 5.424263840999629, + 5.502526098996896, + 5.567715520999627, + 5.619325887997547, + 5.681696723997447, + 5.739977671997622, + 5.799626860996796, + 5.8558119849985815, + 5.904651609998837, + 5.951749944997573, + 6.011433987998316, + 6.0804560119977396 + ], + "chunk_gaps_ms": { + "count": 99, + "p50": 59.68404300074326, + "p95": 78.86902099926374, + "max": 83.59345299686538 + } + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 2, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.0, + "per_stream_tps": 40.8, + "ttft_s_max": 0.18, + "request": { + "ttft_s": 0.131, + "decode_tps": 40.769, + "prefill_tps": null, + "itl_mean_ms": 24.529, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.939, + "round_gap_p50_ms": 59.172, + "round_gap_p95_ms": 80.007, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 24.532, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.57, + "round_gap_p50_ms": 59.115, + "round_gap_p95_ms": 79.995, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 80.17065196379662, + "makespan_s": 6.386376902999473, + "steady_tps": 81.9, + "alone": { + "equal": 6, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 80.97342759140203, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.386376902999473, + "end_to_end_tps": 80.17065196379662, + "timing_valid": true, + "steady_tps": 81.85204257365322, + "per_stream_tps": [ + 41.0, + 40.5 + ], + "ttft_s": [ + 0.17, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.17266030199971283, + "decode_tps": 41.04197373211658, + "prefill_tps": null, + "itl_mean_ms": 24.365299937255937, + "itl_p50_ms": 0.0, + "itl_p95_ms": 74.98608199966839, + "round_gap_p50_ms": 59.22936400020262, + "round_gap_p95_ms": 80.02209700134699, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08760773700123536, + "decode_tps": 40.486713795701014, + "prefill_tps": null, + "itl_mean_ms": 24.699460792152085, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.2803209979902, + "round_gap_p50_ms": 59.11483099771431, + "round_gap_p95_ms": 80.79969700338552, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.53238036470401, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.11833900207421, + "round_gap_p50_ms": 59.11483099771431, + "round_gap_p95_ms": 80.48773699920275, + "round_gap_mean_ms": 61.93818804950022, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30603.83535404, + "first": 30604.008014342, + "last": 30610.221165826, + "end": 30610.221611839, + "elapsed_s": 6.386257799000305, + "chunk_offsets_s": [ + 0.17266030199971283, + 0.2351330190031149, + 0.3042792400010512, + 0.3822524910028733, + 0.44270268800028134, + 0.49274863000027835, + 0.5441322780025075, + 0.6052608820027672, + 0.6665733959998761, + 0.7357326710007328, + 0.7999516600029892, + 0.8746603070030687, + 0.9487608620001993, + 1.0131841790025646, + 1.0682829320030578, + 1.126508745001047, + 1.19446940400303, + 1.261641959001281, + 1.3220130059999065, + 1.388971850003145, + 1.4520739750005305, + 1.512201792000269, + 1.592223889001616, + 1.6783656450024864, + 1.7559227980018477, + 1.8172703689997434, + 1.8734603260018048, + 1.9274186710026697, + 1.9778135180022218, + 2.0262514030000602, + 2.078637039001478, + 2.1431110610028554, + 2.1961360050008807, + 2.2467187310030567, + 2.2999973580008373, + 2.3555721010015986, + 2.4044588100005058, + 2.4624726150032075, + 2.516630810001516, + 2.5758601740017184, + 2.6361295740025525, + 2.6882720030007476, + 2.7408214700008102, + 2.7985651920025703, + 2.8497494910006935, + 2.903043221002008, + 2.9591460980009288, + 3.0132538950019807, + 3.073496609002177, + 3.140476453001611, + 3.213659472003201, + 3.2660507000000507, + 3.3192887360019085, + 3.370105310001236, + 3.4165895440019085, + 3.464316581001185, + 3.518919720001577, + 3.576380259000871, + 3.6329842660015856, + 3.6960097150004003, + 3.7479275750010856, + 3.80629110800146, + 3.881558072000189, + 3.9563033400008862, + 4.038274501999695, + 4.116021631001786, + 4.188549361002515, + 4.244389608000347, + 4.319375690000015, + 4.395557285002724, + 4.457734638002876, + 4.504779681003129, + 4.562750779001362, + 4.634944777000783, + 4.703237711000838, + 4.759534385000734, + 4.819681225002569, + 4.8816630610017455, + 4.938018832002854, + 4.995382962002623, + 5.05624823800099, + 5.102704723001807, + 5.15584183500323, + 5.228103290002764, + 5.308591027001967, + 5.386996874000033, + 5.451522718001797, + 5.506463858000643, + 5.577274484003283, + 5.660738845999731, + 5.743725236003229, + 5.812271879000036, + 5.869096993999847, + 5.929346439002984, + 5.987364584001625, + 6.0421390410010645, + 6.100757464002527, + 6.152966590001597, + 6.203495012003259, + 6.25733058500191, + 6.312476200000674, + 6.385811785999977 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.22936400020262, + "p95": 80.02209700134699, + "max": 86.14175600087037 + } + }, + { + "start": 30603.835234936, + "first": 30603.922842673, + "last": 30610.221205175, + "end": 30610.221280236, + "elapsed_s": 6.38604530000157, + "chunk_offsets_s": [ + 0.08760773700123536, + 0.23543337299997802, + 0.3041596410002967, + 0.3821770020003896, + 0.4425326660020801, + 0.49261919600030524, + 0.5444880520008155, + 0.6056339860006119, + 0.666375536002306, + 0.7360003499998129, + 0.8001766689994838, + 0.8746128160018998, + 0.9484273689995462, + 1.0130758449995483, + 1.0680978300006245, + 1.1264443450018007, + 1.1944827170009376, + 1.2615691499995592, + 1.3218802949995734, + 1.389035264000995, + 1.4524037470000621, + 1.5122590710016084, + 1.592282929999783, + 1.6786069250010769, + 1.7559821750000992, + 1.8173499760014238, + 1.8735206490018754, + 1.9277259080008662, + 1.9778948560015124, + 2.026274472998921, + 2.079067884998949, + 2.143444446999638, + 2.196419853000407, + 2.2467347380006686, + 2.299921271001949, + 2.355805881001288, + 2.404509701002098, + 2.46284083700084, + 2.516682165001839, + 2.5757969959995535, + 2.6362018629988597, + 2.688645797999925, + 2.740885621999041, + 2.7989137770018715, + 2.8498140440024144, + 2.903580800000782, + 2.959415703000559, + 3.0135952359996736, + 3.0738771889991767, + 3.1408681480024825, + 3.2140719880007964, + 3.266112178000185, + 3.3194476240023505, + 3.3702920969990373, + 3.4167624450019503, + 3.4643826890023774, + 3.5189760729990667, + 3.5764288770005805, + 3.63317880500108, + 3.6960655080001743, + 3.7479929460023413, + 3.8064553610020084, + 3.8817356819999986, + 3.9563553869993484, + 4.038306336999085, + 4.1159543140020105, + 4.188513515000523, + 4.244462041999213, + 4.319580381001288, + 4.395718957999634, + 4.457922960998985, + 4.5048368370007665, + 4.562791967000521, + 4.634795444999327, + 4.70343961600156, + 4.759597882999515, + 4.819735498000227, + 4.8818211470024835, + 4.938174531002005, + 4.9955438489996595, + 5.056552412999736, + 5.102989792001608, + 5.155910715999198, + 5.227864790998865, + 5.30866448800225, + 5.3873846320020675, + 5.451587899002334, + 5.506632033000642, + 5.577512772000773, + 5.66095117399891, + 5.743759812001372, + 5.812336819999473, + 5.869252020002023, + 5.929410435001046, + 5.987523037001665, + 6.042203757999232, + 6.100824471999658, + 6.153133386000263, + 6.203572205002274, + 6.257346008998866, + 6.312649177001731, + 6.385970239000017 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.11483099771431, + "p95": 80.79969700338552, + "max": 147.82563599874265 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 80.77120195386148, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.409067802000209, + "end_to_end_tps": 79.88681284354796, + "timing_valid": true, + "steady_tps": 81.64992771173026, + "per_stream_tps": [ + 40.4, + 40.9 + ], + "ttft_s": [ + 0.09, + 0.18 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.09432462399854558, + "decode_tps": 40.388657395547625, + "prefill_tps": null, + "itl_mean_ms": 24.759426643140614, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.85804899665527, + "round_gap_p50_ms": 59.09645800056751, + "round_gap_p95_ms": 79.78237199858995, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17995480099853012, + "decode_tps": 40.937489455693395, + "prefill_tps": null, + "itl_mean_ms": 24.42748720783914, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.34575700285495, + "round_gap_p50_ms": 58.65887900290545, + "round_gap_p95_ms": 79.60025200009113, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.593456925489875, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.47646700206678, + "round_gap_p50_ms": 58.65887900290545, + "round_gap_p95_ms": 79.78237199858995, + "round_gap_mean_ms": 62.09239124752394, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30610.223338442, + "first": 30610.317663066, + "last": 30616.63131686, + "end": 30616.631388838, + "elapsed_s": 6.408050395999453, + "chunk_offsets_s": [ + 0.09432462399854558, + 0.24709530399923096, + 0.3181285239988938, + 0.39607662799971877, + 0.45588028999918606, + 0.5061975409989827, + 0.5573536779993447, + 0.6121657020012208, + 0.6715974700018705, + 0.7416245849999541, + 0.8069971190016076, + 0.8836520350014325, + 0.9569422040003701, + 1.020514378000371, + 1.074785957000131, + 1.135166421001486, + 1.2082902490001288, + 1.2774768140006927, + 1.3389424160013732, + 1.4065669010014972, + 1.469449093001458, + 1.5285455510020256, + 1.6043870020002942, + 1.6861503620020812, + 1.7630084109987365, + 1.8255232090014033, + 1.8815746850013966, + 1.936776724000083, + 1.9862492719985312, + 2.0353882769995835, + 2.089845709000656, + 2.1529122249994543, + 2.212300782000966, + 2.264498625001579, + 2.3172165260002657, + 2.372614865998912, + 2.4222893830010435, + 2.4784856100013712, + 2.5330995000003895, + 2.5889166740016663, + 2.648613067001861, + 2.699551371999405, + 2.753275446000771, + 2.811442974001693, + 2.8624854469999264, + 2.915430068002024, + 2.970873557002051, + 3.024534282001696, + 3.085363922000397, + 3.152139377001731, + 3.2290913889992225, + 3.2843542590017023, + 3.3373574580000422, + 3.3895787530018424, + 3.4364878660016984, + 3.484489164999104, + 3.5384018820004712, + 3.594232371000544, + 3.6483578820007097, + 3.709496018000209, + 3.7614016810002795, + 3.818881962000887, + 3.895070942999155, + 3.969021517001238, + 4.053067191001901, + 4.132226649999211, + 4.209394469999097, + 4.266994353998598, + 4.341783280000527, + 4.41929039200113, + 4.480286376001459, + 4.5264949679985875, + 4.583653085999686, + 4.652137339999172, + 4.718723557001795, + 4.77401318000193, + 4.8340924920012185, + 4.897550959998625, + 4.952009997999994, + 5.006767777002096, + 5.067256000002089, + 5.113772844000778, + 5.166111614998954, + 5.24222495900176, + 5.324855007998849, + 5.404210403001343, + 5.468898391001858, + 5.525322497000161, + 5.593466661001003, + 5.673249032999593, + 5.7584777230003965, + 5.827202209999086, + 5.884477248000621, + 5.944856775000517, + 6.002125776001776, + 6.058466274000239, + 6.116341777000343, + 6.169899404001626, + 6.224136392000219, + 6.280671221000375, + 6.336246016999212, + 6.407978417999402 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.09645800056751, + "p95": 79.78237199858995, + "max": 152.7706800006854 + } + }, + { + "start": 30610.222830644, + "first": 30610.402785445, + "last": 30616.631794683, + "end": 30616.631898446, + "elapsed_s": 6.409067802000209, + "chunk_offsets_s": [ + 0.17995480099853012, + 0.24729910899986862, + 0.31841477299894905, + 0.3969301529978111, + 0.4568843540000671, + 0.5068667359992105, + 0.5580626060000213, + 0.6128968209995946, + 0.6722921729997324, + 0.742339930999151, + 0.8079493499972159, + 0.8842951070000709, + 0.9572682379985054, + 1.0211863399999856, + 1.0755952109975624, + 1.1359579309973924, + 1.2089647889988555, + 1.2779004240001086, + 1.3396829519988387, + 1.407245171998511, + 1.4702717189975374, + 1.5289305980004428, + 1.604688729999907, + 1.6868983229978767, + 1.7637392219985486, + 1.8261806909977167, + 1.8822245279989147, + 1.9374063569994178, + 1.986616497997602, + 2.0361285359977046, + 2.0905253870005254, + 2.1535604989985586, + 2.2126497520002886, + 2.265253183999448, + 2.317941826997412, + 2.3734128719988803, + 2.423017278997577, + 2.4794161959980556, + 2.533914752999408, + 2.5899245820000942, + 2.6493965259978722, + 2.7003040839990717, + 2.754028686998936, + 2.8124024719982117, + 2.863163727997744, + 2.9163296909973724, + 2.9711446080000314, + 3.025335996000649, + 3.086376905997895, + 3.1528310449975834, + 3.22930751199965, + 3.2850925589991675, + 3.338081409998267, + 3.3902581709990045, + 3.437402684998233, + 3.4858205349992204, + 3.5394565659989894, + 3.594895739999629, + 3.649212242999056, + 3.710294939999585, + 3.76227514600032, + 3.8196724129993527, + 3.8957305000003544, + 3.9698977199986984, + 4.053820474997337, + 4.132981168000697, + 4.209637132997159, + 4.26768813200033, + 4.342743152999901, + 4.420091956999386, + 4.481075241998042, + 4.526878292999754, + 4.584564703000069, + 4.652792875000159, + 4.718895527999848, + 4.774837076998665, + 4.834896452997782, + 4.898288209999009, + 4.952625836998777, + 5.0069898209985695, + 5.067623764000018, + 5.113854451999941, + 5.166892083998391, + 5.2423967049981, + 5.325468924998859, + 5.40506917699895, + 5.469143365997297, + 5.526126681997994, + 5.594246996999573, + 5.674273966000328, + 5.759467666997807, + 5.8280131299979985, + 5.8852295459983, + 5.945541818997299, + 6.0031041109978105, + 6.058701404999738, + 6.11704004800049, + 6.170235080000566, + 6.224526079000498, + 6.281041292997543, + 6.3371213899990835, + 6.40896403899751 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 58.65887900290545, + "p95": 79.60025200009113, + "max": 85.19370099747903 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 81.20036834664855, + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 6.366457939999236, + "end_to_end_tps": 80.42148472908335, + "timing_valid": true, + "steady_tps": 82.05919205045454, + "per_stream_tps": [ + 41.1, + 40.6 + ], + "ttft_s": [ + 0.17, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16795377300149994, + "decode_tps": 41.145841920596446, + "prefill_tps": null, + "itl_mean_ms": 24.30379239607753, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.53137600189075, + "round_gap_p50_ms": 59.47166300029494, + "round_gap_p95_ms": 79.9920489989745, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08491770399996312, + "decode_tps": 40.60018417332427, + "prefill_tps": null, + "itl_mean_ms": 24.630430141177403, + "itl_p50_ms": 0.0, + "itl_p95_ms": 76.42279499850702, + "round_gap_p50_ms": 59.378302001277916, + "round_gap_p95_ms": 80.10044799812022, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 2, + "itl_mean_ms": 24.467111268627466, + "itl_p50_ms": 0.0, + "itl_p95_ms": 75.57015800193767, + "round_gap_p50_ms": 59.378302001277916, + "round_gap_p95_ms": 79.99524600018049, + "round_gap_mean_ms": 61.7733997376238, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30616.63294322, + "first": 30616.800896993, + "last": 30622.998364054, + "end": 30622.998839708, + "elapsed_s": 6.365896488001454, + "chunk_offsets_s": [ + 0.16795377300149994, + 0.23163062100138632, + 0.3008925169997383, + 0.37793903700003284, + 0.43843786000070395, + 0.48913159600124345, + 0.539019660998747, + 0.5954869909983245, + 0.6532179349997023, + 0.7223099049988377, + 0.7891271170010441, + 0.8691223630012246, + 0.9429082489987195, + 1.007727383999736, + 1.061783673998434, + 1.1217988710013742, + 1.189943815999868, + 1.2569631869992008, + 1.3145254150003893, + 1.3808427129988559, + 1.443458264999208, + 1.503321757998492, + 1.5788531340003829, + 1.6618300500012992, + 1.7386253049990046, + 1.8057981060010206, + 1.8665348920003453, + 1.919956812998862, + 1.9696024199984095, + 2.016353283001081, + 2.0697460299998056, + 2.1318081339995842, + 2.186779435000062, + 2.2375062339997385, + 2.289157195998996, + 2.343210741000803, + 2.3936784340003214, + 2.450079464000737, + 2.504706621999503, + 2.559923181001068, + 2.6178598400001647, + 2.669724595001753, + 2.721878109001409, + 2.7825826959997357, + 2.8364469469997857, + 2.8906543029988825, + 2.9457032649988832, + 2.99984316300106, + 3.0600883020015317, + 3.1270390989993757, + 3.19867735400112, + 3.252187525999034, + 3.3048097350001626, + 3.355708555998717, + 3.4031294859996706, + 3.450981147998391, + 3.505532849998417, + 3.5593874660007714, + 3.6138645530008944, + 3.6749265719990944, + 3.7266957409992756, + 3.7897646839992376, + 3.869756732998212, + 3.9432750689993554, + 4.02541520199884, + 4.1038382629994885, + 4.1759439169982215, + 4.232049198999448, + 4.305146067999885, + 4.381759263000276, + 4.441797937000956, + 4.489524805001565, + 4.547893340000883, + 4.615006902000459, + 4.6821986159993685, + 4.736800966998999, + 4.800406242000463, + 4.865627890001633, + 4.923099269999511, + 4.979234605001693, + 5.040087781999318, + 5.0863585519982735, + 5.139413059001527, + 5.20878903799894, + 5.2880039349984145, + 5.367194047001249, + 5.432088338999165, + 5.488409492001665, + 5.556898301998444, + 5.63787100600166, + 5.722052949000499, + 5.794093717999203, + 5.853565380999498, + 5.915264412000397, + 5.973474072001409, + 6.028067341998394, + 6.08413023400135, + 6.1370187110005645, + 6.186115937998693, + 6.239990261998173, + 6.294470216998889, + 6.36542083400127 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.47166300029494, + "p95": 79.9920489989745, + "max": 84.18194299883908 + } + }, + { + "start": 30616.632841234, + "first": 30616.717758938, + "last": 30622.998518624, + "end": 30622.999299174, + "elapsed_s": 6.366457939999236, + "chunk_offsets_s": [ + 0.08491770399996312, + 0.23153555699900608, + 0.3004507490004471, + 0.3779122339983587, + 0.4384528019982099, + 0.48895930299840984, + 0.5387751679991197, + 0.5957323539987556, + 0.6530986600009783, + 0.722352131997468, + 0.7890368730004411, + 0.8691373209985613, + 0.9433537679979054, + 1.0080232660002366, + 1.0621976479997102, + 1.1217418039996119, + 1.189890050998656, + 1.2569818449992454, + 1.314885838997725, + 1.3808649739985412, + 1.4437938209994172, + 1.5033045159980247, + 1.5788746739999624, + 1.66207404799934, + 1.7384968429978471, + 1.8062194239973905, + 1.8663349179987563, + 1.9196091109988629, + 1.9694308689977333, + 2.016259086998616, + 2.0699280129992985, + 2.1323331799976586, + 2.1871025459986413, + 2.237959170000977, + 2.2894438000002992, + 2.3434964810003294, + 2.393546805000369, + 2.450415768998937, + 2.505003618000046, + 2.5602269819974026, + 2.6182999669981655, + 2.6695912850009336, + 2.721679870999651, + 2.78245409399824, + 2.8364167869985977, + 2.8906299070004025, + 2.94600114199784, + 3.0002294730002177, + 3.060302114998194, + 3.127481595998688, + 3.1987037809994945, + 3.2520153429977654, + 3.305227215998457, + 3.3554769049987954, + 3.4030067959974986, + 3.450950685997668, + 3.5058740349995787, + 3.559827762997884, + 3.6135861969996768, + 3.6749043870004243, + 3.7266560069983825, + 3.789636517998588, + 3.869492306999746, + 3.942967602997669, + 4.025370808998559, + 4.103422861997387, + 4.175422512998921, + 4.231929730998672, + 4.304921964998357, + 4.382222820000607, + 4.44210430900057, + 4.489326224000251, + 4.548283116000675, + 4.61537362099989, + 4.6819797809985175, + 4.736786277000647, + 4.800718361999316, + 4.865341582000838, + 4.923413759999676, + 4.9795466139985365, + 5.039950749000127, + 5.086157761998038, + 5.139259037998272, + 5.208761492998747, + 5.287867433999054, + 5.367129356000078, + 5.431975598999998, + 5.488242056999297, + 5.556728128998657, + 5.638216393999755, + 5.721996375999879, + 5.7940612099991995, + 5.853439512000477, + 5.915649443999428, + 5.973738730997866, + 6.02805196600093, + 6.084072075998847, + 6.1372552559987525, + 6.186289717999898, + 6.2399023580001085, + 6.294719151999743, + 6.365677390000201 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 59.378302001277916, + "p95": 80.10044799812022, + "max": 146.61785299904295 + } + } + ], + "equal_alone": [ + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 141.0, + "per_stream_tps": 35.9, + "ttft_s_max": 0.35, + "request": { + "ttft_s": 0.22, + "decode_tps": 35.861, + "prefill_tps": null, + "itl_mean_ms": 27.886, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.145, + "round_gap_p50_ms": 65.224, + "round_gap_p95_ms": 92.992, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 27.88, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.886, + "round_gap_p50_ms": 65.099, + "round_gap_p95_ms": 92.998, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 139.86837248625437, + "makespan_s": 7.321169052000187, + "steady_tps": 145.1, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 141.00045171326343, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.321169052000187, + "end_to_end_tps": 139.86837248625437, + "timing_valid": true, + "steady_tps": 145.12723405851006, + "per_stream_tps": [ + 36.5, + 35.7, + 36.1, + 35.3 + ], + "ttft_s": [ + 0.33, + 0.17, + 0.25, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.3347470560001966, + "decode_tps": 36.502875456409754, + "prefill_tps": null, + "itl_mean_ms": 27.39510209802949, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.56657400206313, + "round_gap_p50_ms": 65.09918600204401, + "round_gap_p95_ms": 93.37367600164725, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16986090399950626, + "decode_tps": 35.66448806891923, + "prefill_tps": null, + "itl_mean_ms": 28.03909586666622, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.12652800022624, + "round_gap_p50_ms": 65.01800099795219, + "round_gap_p95_ms": 93.9697210014856, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25117966499965405, + "decode_tps": 36.0758088627005, + "prefill_tps": null, + "itl_mean_ms": 27.719406203915224, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.3213119994034, + "round_gap_p50_ms": 65.34952599758981, + "round_gap_p95_ms": 93.83959499973571, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.0870434030002798, + "decode_tps": 35.254475319231375, + "prefill_tps": null, + "itl_mean_ms": 28.365193098037636, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.26288599791587, + "round_gap_p50_ms": 65.53321200044593, + "round_gap_p95_ms": 93.57597999769496, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.879699316662144, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.88561099916114, + "round_gap_p50_ms": 65.09918600204401, + "round_gap_p95_ms": 93.83959499973571, + "round_gap_mean_ms": 70.38933985889948, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30623.001221291, + "first": 30623.335968347, + "last": 30630.321719382, + "end": 30630.321825653, + "elapsed_s": 7.320604362001177, + "chunk_offsets_s": [ + 0.3347470560001966, + 0.40232629000092857, + 0.4856613929987361, + 0.5759294870003941, + 0.644264006998128, + 0.6991048970012343, + 0.7544035089995305, + 0.8155036309981369, + 0.8790880480009946, + 0.9589430730011372, + 1.0306550370005425, + 1.1171333029997186, + 1.1987415679977858, + 1.2753334759981954, + 1.3391743059983128, + 1.4037698969987105, + 1.485557545998745, + 1.5657867090012587, + 1.629459121999389, + 1.7061164179976913, + 1.7767989229978411, + 1.8418981089998852, + 1.9279209339983936, + 2.02102533499783, + 2.1075919089998933, + 2.1795633809997526, + 2.2432342840002093, + 2.3035972069992567, + 2.3579553569979907, + 2.4111477010010276, + 2.4731997970011435, + 2.5447176109992142, + 2.6033527920008055, + 2.6582528660001117, + 2.715368536999449, + 2.7779899669985753, + 2.834239317999163, + 2.8981630640009826, + 2.96069066400014, + 3.0231249559983553, + 3.0887060380009643, + 3.1457758260003175, + 3.202769741001248, + 3.2702117640001234, + 3.3255182899993088, + 3.3824398219985596, + 3.4471666120007285, + 3.5089930089998234, + 3.573199160000513, + 3.6490309439977864, + 3.7330810179992113, + 3.7900969889997214, + 3.848699943999236, + 3.907775149000372, + 3.960279401999287, + 4.01413426399813, + 4.072192684998299, + 4.133539208000002, + 4.191250001000299, + 4.259998416000599, + 4.317719935999776, + 4.38017837499865, + 4.473552051000297, + 4.561193542998808, + 4.655374480000319, + 4.745098241997766, + 4.828837271001248, + 4.894108385000436, + 4.973708863999491, + 5.06043093399785, + 5.128504992000671, + 5.180673643000773, + 5.245287535999523, + 5.321111893001216, + 5.396122196998476, + 5.465519424000377, + 5.532151520998013, + 5.603309689999151, + 5.66495498400036, + 5.729334412000753, + 5.798220000000583, + 5.8515622619997885, + 5.911769039998035, + 5.990371347001201, + 6.084021355000004, + 6.174541616001079, + 6.251326858000539, + 6.3150765120008145, + 6.389021815000888, + 6.483285584999976, + 6.583649953998247, + 6.661649977999332, + 6.724516122998466, + 6.794457432999479, + 6.860792488998413, + 6.9203650599993125, + 6.986348659000214, + 7.046361971999431, + 7.102383720000944, + 7.162683089998609, + 7.224607421998371, + 7.320498090997717 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.09918600204401, + "p95": 93.37367600164725, + "max": 100.36436899827095 + } + }, + { + "start": 30623.000887799, + "first": 30623.170748703, + "last": 30630.320718149, + "end": 30630.321187579, + "elapsed_s": 7.320299779999914, + "chunk_offsets_s": [ + 0.16986090399950626, + 0.4027863490009622, + 0.4863620649994118, + 0.5762810710002668, + 0.6448041519979597, + 0.6995250399995712, + 0.7546009549987502, + 0.8160345040014363, + 0.8796948599992902, + 0.9590996150000137, + 1.0300992769989534, + 1.1175442870007828, + 1.199023778000992, + 1.2758802309981547, + 1.3378875749986037, + 1.4043626509992464, + 1.4855589590006275, + 1.5660465520013531, + 1.6302229659995646, + 1.7066733239989844, + 1.7767977750008868, + 1.8416933589978726, + 1.9280748199998925, + 2.021613378001348, + 2.107717087998026, + 2.179696260998753, + 2.243597699998645, + 2.30386835399986, + 2.358915564000199, + 2.4111943319985585, + 2.4733171629995923, + 2.5454679370013764, + 2.603927371001191, + 2.6593006670009345, + 2.71518332100095, + 2.7784713319997536, + 2.8347230190011032, + 2.8983808170014527, + 2.961480529000255, + 3.0233683420010493, + 3.0893304210003407, + 3.1462929240005906, + 3.2028671609987214, + 3.2701362410007278, + 3.3256305520008027, + 3.3828219860006357, + 3.447839986998588, + 3.5091460490002646, + 3.573574455000198, + 3.6494543019980483, + 3.733367823999288, + 3.7902961399995547, + 3.849131290000514, + 3.9080475629998546, + 3.9603946759998507, + 4.014500592998957, + 4.072395966999466, + 4.1339065930005745, + 4.191490891000285, + 4.260057686999062, + 4.317974289999256, + 4.380583496000327, + 4.474052016001224, + 4.56117854400145, + 4.6554969909993815, + 4.745284361000813, + 4.829368132999662, + 4.89405274100136, + 4.974121412000386, + 5.061007022999547, + 5.128568926000298, + 5.181068724999932, + 5.245459424000728, + 5.320818583000801, + 5.396062134997919, + 5.4661423260004085, + 5.532344270999602, + 5.604009440001391, + 5.6654654490012035, + 5.729822015000536, + 5.798590394999337, + 5.851703942000313, + 5.911985292997997, + 5.990453903999878, + 6.084423625001364, + 6.174607297998591, + 6.25161206499979, + 6.315131595998537, + 6.388888688998122, + 6.483674887000234, + 6.583813424000255, + 6.661620834998757, + 6.725081370997941, + 6.794526128000143, + 6.861217154000769, + 6.9203224730008515, + 6.986644128999615, + 7.045902081998065, + 7.102695760000643, + 7.1629368549984065, + 7.225031239999225, + 7.319830349999393 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.01800099795219, + "p95": 93.9697210014856, + "max": 232.92544500145596 + } + }, + { + "start": 30623.001742945, + "first": 30623.25292261, + "last": 30630.321371192, + "end": 30630.321411552, + "elapsed_s": 7.319668606996856, + "chunk_offsets_s": [ + 0.25117966499965405, + 0.40185763299814425, + 0.48533534499802045, + 0.575369165999291, + 0.6440630209981464, + 0.6988835129996005, + 0.7538557889965887, + 0.8150333079975098, + 0.8786528040000121, + 0.9583789589996741, + 1.0294246999983443, + 1.1167460119977477, + 1.198203198997362, + 1.2749789419976878, + 1.337342748996889, + 1.4035836679977365, + 1.4848519759980263, + 1.565414194999903, + 1.6291535809978086, + 1.7060042220000469, + 1.7761238219973166, + 1.8416275799972937, + 1.9278839659964433, + 2.020545275998302, + 2.1069748959998833, + 2.17890395699942, + 2.242766921997827, + 2.303117868999834, + 2.3583133359970816, + 2.4104147399993963, + 2.472400023998489, + 2.544432095997763, + 2.603363743997761, + 2.6581048169973656, + 2.7146879469983105, + 2.777424556999904, + 2.833684650999203, + 2.8976032089994987, + 2.9608469649974722, + 3.0227994759989087, + 3.0881490019964986, + 3.1454027149993635, + 3.2020519609977782, + 3.269732328997634, + 3.3245049759971153, + 3.381714965998981, + 3.4465355279971845, + 3.508172250996722, + 3.5721353979970445, + 3.648690958998486, + 3.732603199998266, + 3.78966665599728, + 3.848416055996495, + 3.9072944649997225, + 3.9596786579968466, + 4.0136735279993445, + 4.071716546997777, + 4.132873830996687, + 4.190792082998087, + 4.259164564999082, + 4.316938518997631, + 4.379768855997099, + 4.473254313998041, + 4.560744366997824, + 4.654924021997431, + 4.744311316997482, + 4.828404343999864, + 4.893361007998465, + 4.973507742997754, + 5.059622318996844, + 5.12786593799683, + 5.180304482997599, + 5.24430647799818, + 5.320198221997998, + 5.395265584997105, + 5.465415595997911, + 5.531218493997585, + 5.603346347998013, + 5.6644004299996595, + 5.729160684997623, + 5.7978956819970335, + 5.851007387998834, + 5.911194905998855, + 5.989983517996734, + 6.08382311299647, + 6.173828742998012, + 6.250851375996717, + 6.314588206998451, + 6.388087110997731, + 6.4829005419996975, + 6.582904660997883, + 6.660961551999208, + 6.723962704996666, + 6.793707771998015, + 6.860394267998345, + 6.919421907998185, + 6.9859401599969715, + 7.045201779998024, + 7.101803887999267, + 7.162131385997782, + 7.224144650997914, + 7.319628246998036 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.34952599758981, + "p95": 93.83959499973571, + "max": 150.6779679984902 + } + }, + { + "start": 30623.000656601, + "first": 30623.087700004, + "last": 30630.320824244, + "end": 30630.321015461, + "elapsed_s": 7.320358859997214, + "chunk_offsets_s": [ + 0.0870434030002798, + 0.4030933109970647, + 0.48593940699720406, + 0.5763862149979104, + 0.6446143419998407, + 0.6995462699997006, + 0.7546609639975941, + 0.8156160019971139, + 0.8794756919996871, + 0.9590839589982352, + 1.030018953999388, + 1.117409015998419, + 1.1987328010000056, + 1.276286442996934, + 1.338300254999922, + 1.404149358997529, + 1.4857006089987408, + 1.5661538269996527, + 1.6303913859992463, + 1.7064586709966534, + 1.7767239999993762, + 1.8418074410001282, + 1.929070326998044, + 2.02144669900008, + 2.1077962339986698, + 2.179632721999951, + 2.2434457979979925, + 2.303827566000109, + 2.3581099699986225, + 2.41119478699693, + 2.473680479997711, + 2.545112109997717, + 2.60357714699785, + 2.6583158499997808, + 2.715526863998093, + 2.7781675229998655, + 2.8344422399968607, + 2.898117264998291, + 2.9610079789999872, + 3.0231080719968304, + 3.089064931999019, + 3.1462404999983846, + 3.203408542998659, + 3.270586299997376, + 3.3254851879974012, + 3.3826645019980788, + 3.4481977139985247, + 3.5095061779975367, + 3.5733714059970225, + 3.6491659819985216, + 3.733284931997332, + 3.790134898998076, + 3.8490532849973533, + 3.9080342870001914, + 3.960182618997351, + 4.014433250998991, + 4.0724728440000035, + 4.133800073999737, + 4.1921608929988, + 4.260162404996663, + 4.3179398009997385, + 4.380979789999401, + 4.474555769997096, + 4.561537965997559, + 4.6554135589976795, + 4.745163001996843, + 4.829029507000087, + 4.893981255998369, + 4.97479894999924, + 5.060359791998053, + 5.129284694998205, + 5.180995782997343, + 5.245918377997441, + 5.321418632000132, + 5.396510794998903, + 5.465975134997279, + 5.532633984999848, + 5.603320405996783, + 5.665325858997676, + 5.729736073997628, + 5.798579301997961, + 5.851811769996857, + 5.911950073998014, + 5.990863721999631, + 6.084190862999094, + 6.1745824059980805, + 6.2515676409966545, + 6.315501245997439, + 6.389470193997113, + 6.483504066000023, + 6.5836643439979525, + 6.66197264999937, + 6.724927679999382, + 6.79459807999956, + 6.861089499998343, + 6.920264059997862, + 6.986662416999025, + 7.0465743569984625, + 7.10270106399912, + 7.162968894997903, + 7.225077351999062, + 7.320167642999877 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.53321200044593, + "p95": 93.57597999769496, + "max": 316.0499079967849 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 140.41839710151118, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.348283667000942, + "end_to_end_tps": 139.3522686935037, + "timing_valid": true, + "steady_tps": 144.97871692056677, + "per_stream_tps": [ + 36.5, + 36.0, + 35.1, + 35.6 + ], + "ttft_s": [ + 0.35, + 0.27, + 0.08, + 0.19 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.35434668800007785, + "decode_tps": 36.47036067353321, + "prefill_tps": null, + "itl_mean_ms": 27.419525925492337, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.02528100184281, + "round_gap_p50_ms": 64.78475100084324, + "round_gap_p95_ms": 92.46737799912808, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.27028798399987863, + "decode_tps": 36.03876877418863, + "prefill_tps": null, + "itl_mean_ms": 27.74789578039667, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.56490100131487, + "round_gap_p50_ms": 64.64764100019238, + "round_gap_p95_ms": 92.99792800084106, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08379917300044326, + "decode_tps": 35.109138123742454, + "prefill_tps": null, + "itl_mean_ms": 28.482613172544756, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.48939300130587, + "round_gap_p50_ms": 64.82971900186385, + "round_gap_p95_ms": 92.98665799724404, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.18843234599989955, + "decode_tps": 35.61703305137038, + "prefill_tps": null, + "itl_mean_ms": 28.076454278426336, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.88553200065508, + "round_gap_p50_ms": 64.77684200217482, + "round_gap_p95_ms": 93.48593999675359, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.931622289215024, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.09767199979979, + "round_gap_p50_ms": 64.7297779978544, + "round_gap_p95_ms": 92.99792800084106, + "round_gap_mean_ms": 70.52043251237457, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30630.324391748, + "first": 30630.678738436, + "last": 30637.670717547, + "end": 30637.671268457, + "elapsed_s": 7.346876709001663, + "chunk_offsets_s": [ + 0.35434668800007785, + 0.4210763289993338, + 0.499272908000421, + 0.5893163260006986, + 0.6583712249994278, + 0.7128593810011807, + 0.7687573789989983, + 0.8308144399998127, + 0.8947487350014853, + 0.9740464110000175, + 1.0450129480013857, + 1.1365480960012064, + 1.2215374440020241, + 1.2961130190014956, + 1.3591895200006547, + 1.4249791160000314, + 1.5023433050009771, + 1.5804489090005518, + 1.644220595000661, + 1.7223162100017362, + 1.7930655150012171, + 1.8582675840007141, + 1.9486492620017088, + 2.041116640000837, + 2.131424117000279, + 2.2071660879992123, + 2.271730528998887, + 2.3325334540022595, + 2.386727017001249, + 2.438814036999247, + 2.498237423002138, + 2.568861145999108, + 2.62840266800049, + 2.682281667999632, + 2.7391934180013777, + 2.801373436999711, + 2.857847438001045, + 2.922081787000934, + 2.9839553620004153, + 3.04578720200152, + 3.1162115980005183, + 3.1725185780014726, + 3.2318927260021155, + 3.299043695002183, + 3.3524387620018388, + 3.411552973000653, + 3.4729373340014718, + 3.532464998999785, + 3.5966617810008756, + 3.672889789999317, + 3.75635906000025, + 3.812857898999937, + 3.8720930999988923, + 3.9319695640006103, + 3.9866441400008625, + 4.038704757000232, + 4.0994738539993705, + 4.1634452319995034, + 4.223401676001231, + 4.291127590000542, + 4.34960439200222, + 4.414408450000337, + 4.5009148019998975, + 4.58794008300174, + 4.682287890002044, + 4.771330767001928, + 4.855816044000676, + 4.9194415689999005, + 5.002274710001075, + 5.088615592001588, + 5.1620754249997844, + 5.216115657000046, + 5.280999442999018, + 5.35373147700011, + 5.428736075999041, + 5.492706351000379, + 5.557491102001222, + 5.629081040002347, + 5.6894682080019265, + 5.751669804001722, + 5.821259200001805, + 5.873396274000697, + 5.931056979999994, + 6.01278098200055, + 6.107334805001301, + 6.204140469999402, + 6.280804443998932, + 6.3448108970005705, + 6.419069825999031, + 6.506562291000591, + 6.603604103998805, + 6.681002853001701, + 6.745036268999684, + 6.818272111999249, + 6.8827868010012025, + 6.945187419001741, + 7.01105083800212, + 7.070863188000658, + 7.130840584999532, + 7.19265538000036, + 7.25380544699874, + 7.346325799000624 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 64.78475100084324, + "p95": 92.46737799912808, + "max": 97.04181299821357 + } + }, + { + "start": 30630.32480609, + "first": 30630.595094074, + "last": 30637.670807498, + "end": 30637.671046454, + "elapsed_s": 7.3462403640005505, + "chunk_offsets_s": [ + 0.27028798399987863, + 0.42106158399838023, + 0.49877821499831043, + 0.5894260479981313, + 0.6583559299979242, + 0.7122063559982053, + 0.7686741409997921, + 0.8303114219997951, + 0.8945755409986305, + 0.9738631309992343, + 1.0448183810003684, + 1.136731508999219, + 1.220957917998021, + 1.2960006930006784, + 1.3590142770008242, + 1.424933851998503, + 1.5021347599977162, + 1.580629048999981, + 1.6445820230001118, + 1.722443327998917, + 1.7930079520010622, + 1.8590056569992157, + 1.9484546769999724, + 2.04094107399942, + 2.1309387580004113, + 2.206914845999563, + 2.2715624869997555, + 2.3319323329997133, + 2.3866988150002726, + 2.438618667998526, + 2.498110329001065, + 2.5687252309980977, + 2.6280688820006617, + 2.682078179997916, + 2.739000159999705, + 2.8012547319995065, + 2.857550489999994, + 2.9215887770005793, + 2.98347319000095, + 3.045027051000943, + 3.1159945180006616, + 3.1727842959990085, + 3.231798226999672, + 3.298743352999736, + 3.352314933999878, + 3.4114800540010037, + 3.47303059800106, + 3.5322974160008016, + 3.596592411999154, + 3.672260097999242, + 3.7561616050006705, + 3.812894539998524, + 3.871739111000352, + 3.9317432770003506, + 3.986541264999687, + 4.038673889001075, + 4.099459539000236, + 4.16360141000041, + 4.223257945999649, + 4.290441131000989, + 4.349286422999285, + 4.414435406000848, + 4.500893006999831, + 4.587735998000426, + 4.68208356499963, + 4.771056965000753, + 4.8556230690010125, + 4.91939593799907, + 5.001609449998796, + 5.088007546000881, + 5.16188193799826, + 5.215777964000154, + 5.280207714000426, + 5.353171617000044, + 5.428699953998148, + 5.49261494700113, + 5.557344724998984, + 5.62909562799905, + 5.689383095999801, + 5.751543710000988, + 5.820463904998178, + 5.872584938999353, + 5.930914365999342, + 6.012651349999942, + 6.106746402998397, + 6.203277220000018, + 6.280708446000062, + 6.344077506000758, + 6.418778332998045, + 6.50634323399936, + 6.602969391999068, + 6.680738844999723, + 6.744838103000802, + 6.8179906549994485, + 6.882528682999691, + 6.944558565999614, + 7.0104751479993865, + 7.070737779999035, + 7.130572253998253, + 7.191627985001105, + 7.253003480000189, + 7.34600140800103 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 64.64764100019238, + "p95": 92.99792800084106, + "max": 150.7735999985016 + } + }, + { + "start": 30630.323777929, + "first": 30630.407577102, + "last": 30637.670643461, + "end": 30637.670825828, + "elapsed_s": 7.347047899002064, + "chunk_offsets_s": [ + 0.08379917300044326, + 0.4216252139995049, + 0.49967931100036367, + 0.5907446660021378, + 0.658774473999074, + 0.7135345310016419, + 0.7694157360019744, + 0.8314875889991526, + 0.8950210110015178, + 0.9744168409997656, + 1.0457152019989735, + 1.1372716260011657, + 1.2222141470010683, + 1.2963814380018448, + 1.3591230400015775, + 1.4252763030017377, + 1.502328270002181, + 1.58112202600023, + 1.6445640519996232, + 1.7226056680010515, + 1.7934103490006237, + 1.8587860680017911, + 1.9488227990004816, + 2.0417936150006426, + 2.131614415000513, + 2.207614292001381, + 2.272155231999932, + 2.3330250430008164, + 2.3874043120013084, + 2.4397824260013294, + 2.4987805939999816, + 2.56935726699885, + 2.6289374660009344, + 2.682653367999592, + 2.739628384999378, + 2.801385176000622, + 2.8583761699992465, + 2.922432691000722, + 2.9842206190005527, + 3.0465956260013627, + 3.1162781679995533, + 3.1735745480000332, + 3.2323263170001155, + 3.2994567300011113, + 3.3528056110008038, + 3.412102180001966, + 3.4732911540013447, + 3.5326658560006763, + 3.5975183570008085, + 3.6731044779990043, + 3.756808229001763, + 3.8137914930011902, + 3.872486590000335, + 3.9325179869993008, + 3.9871755940002913, + 4.039445541002351, + 4.100002282000787, + 4.164684129002126, + 4.224062944998877, + 4.291639945000497, + 4.349996953998925, + 4.414902123000502, + 4.501248135002243, + 4.588323250001849, + 4.682624969002063, + 4.77179951600192, + 4.856162967000273, + 4.919840952999948, + 5.002560433000326, + 5.089603210999485, + 5.162767894002172, + 5.216601119002007, + 5.281025537999085, + 5.354009819999192, + 5.429198853998969, + 5.493113708998862, + 5.557943428000726, + 5.629504119002377, + 5.689916515999357, + 5.752083565999783, + 5.821389116001228, + 5.873539525000524, + 5.931476745001419, + 6.013504667000234, + 6.108005246001994, + 6.204463048001344, + 6.28113798800041, + 6.345186366001144, + 6.419475919999968, + 6.506965313001274, + 6.603532479999558, + 6.681415736002236, + 6.745438603000366, + 6.818653778998851, + 6.883147528998961, + 6.945285915000568, + 7.011390435000067, + 7.0718110039997555, + 7.13151296200158, + 7.1925120710002375, + 7.253878874002112, + 7.346865531999356 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 64.82971900186385, + "p95": 92.98665799724404, + "max": 337.82604099906166 + } + }, + { + "start": 30630.323654352, + "first": 30630.512086698, + "last": 30637.671582539, + "end": 30637.671938019, + "elapsed_s": 7.348283667000942, + "chunk_offsets_s": [ + 0.18843234599989955, + 0.4220478800016281, + 0.500052343999414, + 0.5902969829985523, + 0.6597514259992749, + 0.7137156620010501, + 0.7697462329997506, + 0.8317759019992081, + 0.8960019820005982, + 0.9749258260017086, + 1.0460552730000927, + 1.137566647001222, + 1.2223889699998836, + 1.2970424790000834, + 1.3595509539991326, + 1.4260112430019944, + 1.5028301419988566, + 1.5814567470006295, + 1.644643380001071, + 1.722948089998681, + 1.7937543070001993, + 1.8588512120004452, + 1.9491774509988318, + 2.0416298279997136, + 2.1320534990009037, + 2.207804115001636, + 2.272343886001181, + 2.3332294339998043, + 2.387566711000545, + 2.4393267929990543, + 2.4988406160009617, + 2.569552387998556, + 2.6292736579998746, + 2.6827346659993054, + 2.7397033749984985, + 2.8015664810009184, + 2.858545260001847, + 2.9226021819995367, + 2.9844835530020646, + 3.046427823999693, + 3.1165167210019717, + 3.173517528000957, + 3.23281281600066, + 3.300001150000753, + 3.3526719250003225, + 3.4118094609984837, + 3.4734694969993143, + 3.532988118000503, + 3.597138442000869, + 3.67368454499956, + 3.75673155199911, + 3.8135049730008177, + 3.8727895929987426, + 3.9324439199990593, + 3.98757578599907, + 4.039388155000779, + 4.1005520280014025, + 4.164445443999284, + 4.2248123370009125, + 4.2916439850014285, + 4.350270516999444, + 4.4153660949996265, + 4.501965871000721, + 4.588839677999204, + 4.6826513090018125, + 4.7721423789989785, + 4.856704081001226, + 4.920409073001792, + 5.002887127000577, + 5.089984799000376, + 5.162974459999532, + 5.216651601000194, + 5.281679144001828, + 5.354397619001247, + 5.4298121979991265, + 5.493672987999162, + 5.5584498300013365, + 5.630182084001717, + 5.690444578001916, + 5.752619857001264, + 5.8220693549992575, + 5.873936878000677, + 5.931993809001142, + 6.01399219599989, + 6.108196329998464, + 6.205183835998469, + 6.2820082460020785, + 6.34514731300078, + 6.419861315000162, + 6.507746847000817, + 6.603900504000194, + 6.682037523001782, + 6.746134540000639, + 6.8192224980011815, + 6.883748377000302, + 6.9457605380011955, + 7.011583327999688, + 7.071724457000528, + 7.131772049000574, + 7.193107396000414, + 7.254442247001862, + 7.347928186998615 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 64.77684200217482, + "p95": 93.48593999675359, + "max": 233.61553400172852 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 141.09225719932385, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 7.317232661000162, + "end_to_end_tps": 139.94361631519226, + "timing_valid": true, + "steady_tps": 145.33622938295244, + "per_stream_tps": [ + 36.1, + 35.3, + 36.6, + 35.7 + ], + "ttft_s": [ + 0.26, + 0.09, + 0.34, + 0.17 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.25675243099976797, + "decode_tps": 36.119536697335946, + "prefill_tps": null, + "itl_mean_ms": 27.68584792156973, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.16366899898276, + "round_gap_p50_ms": 66.07665899718995, + "round_gap_p95_ms": 91.24358500048402, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.0872541069984436, + "decode_tps": 35.27384728782094, + "prefill_tps": null, + "itl_mean_ms": 28.349615278435238, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.04597899850341, + "round_gap_p50_ms": 65.95197200294933, + "round_gap_p95_ms": 91.05327900033444, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.34073620100025437, + "decode_tps": 36.55835800740043, + "prefill_tps": null, + "itl_mean_ms": 27.353526101953822, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.47591800036025, + "round_gap_p50_ms": 65.79479900028673, + "round_gap_p95_ms": 90.74460299962084, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16969990100187715, + "decode_tps": 35.6836526278966, + "prefill_tps": null, + "itl_mean_ms": 28.024036956862, + "itl_p50_ms": 0.0, + "itl_p95_ms": 87.07128400055808, + "round_gap_p50_ms": 65.9747059980873, + "round_gap_p95_ms": 91.1935350013664, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 27.853256564705198, + "itl_p50_ms": 0.0, + "itl_p95_ms": 86.82465799938655, + "round_gap_p50_ms": 65.8858489987324, + "round_gap_p95_ms": 91.1935350013664, + "round_gap_mean_ms": 70.32257845544382, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30637.673756096, + "first": 30637.930508527, + "last": 30644.990399747, + "end": 30644.990854368, + "elapsed_s": 7.317098272000294, + "chunk_offsets_s": [ + 0.25675243099976797, + 0.4079957150024711, + 0.48619031999987783, + 0.5750319050021062, + 0.6448992059995362, + 0.6997359880006115, + 0.7582031589990947, + 0.8243647689996578, + 0.8885098980026669, + 0.9690293330022541, + 1.0399283910010126, + 1.1270920599999954, + 1.2100394970002526, + 1.2844060419993184, + 1.3468625470013649, + 1.4123523570015095, + 1.4893930530015496, + 1.5664876170012576, + 1.630176560000109, + 1.7082877270004246, + 1.7825688060001994, + 1.8510247049998725, + 1.9388469110017468, + 2.030643501002487, + 2.1159907990004285, + 2.190348799002095, + 2.254508326001087, + 2.313039557000593, + 2.3674440440008766, + 2.4182694000010088, + 2.476867415000015, + 2.5448575090013037, + 2.6063189600026817, + 2.6590403340014745, + 2.7155225759997847, + 2.7833724560005066, + 2.8423346990020946, + 2.906727508001495, + 2.971065746001841, + 3.034502999002143, + 3.100579657999333, + 3.1561095870019926, + 3.214012496999203, + 3.2810511530005897, + 3.3358708010018745, + 3.394654679999803, + 3.453967756999191, + 3.512306045002333, + 3.5773579990018334, + 3.651322216999688, + 3.735041583000566, + 3.7970728979998967, + 3.8578283980023116, + 3.917027389001305, + 3.9702910390005854, + 4.022127314001409, + 4.079221306001273, + 4.138740895999945, + 4.197581693002576, + 4.26706623499922, + 4.323986105002405, + 4.388181902002543, + 4.475160730002244, + 4.558705275001557, + 4.654806295002345, + 4.7449833570026385, + 4.8344921190000605, + 4.900590496999939, + 4.982552404999296, + 5.067308306999621, + 5.136643288002233, + 5.189211125001748, + 5.254371931001515, + 5.328282297999976, + 5.403238801001862, + 5.467346797002392, + 5.533689240000967, + 5.603202104000957, + 5.663172605, + 5.725692309002625, + 5.7990467050003645, + 5.853471760001412, + 5.912991891000274, + 5.994313164999767, + 6.084734421001485, + 6.175978006001969, + 6.253766218000237, + 6.316500208002253, + 6.391496869000548, + 6.479741580002155, + 6.5783879830014484, + 6.657064745002572, + 6.719543514001998, + 6.79444893300024, + 6.860990699002286, + 6.920598521999636, + 6.986158782001439, + 7.047103087999858, + 7.101770200999454, + 7.159435516001395, + 7.220965340999101, + 7.316643651000049 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 66.07665899718995, + "p95": 91.24358500048402, + "max": 151.24328400270315 + } + }, + { + "start": 30637.673833272, + "first": 30637.761087379, + "last": 30644.990239275, + "end": 30644.990430881, + "elapsed_s": 7.316597608998563, + "chunk_offsets_s": [ + 0.0872541069984436, + 0.40806610199797433, + 0.48600291199909407, + 0.5756660179977189, + 0.6449772609994397, + 0.6993905319977785, + 0.7576021009990654, + 0.8240743809983542, + 0.8880515199998626, + 0.9694391569973959, + 1.0403745849980623, + 1.1271578989981208, + 1.2094806559980498, + 1.2845955589982623, + 1.346647349997511, + 1.4124171399998886, + 1.4897303879988613, + 1.5662289899992174, + 1.6298055409970402, + 1.7078690039998037, + 1.7823504230000253, + 1.851023870996869, + 1.9384568739988026, + 2.0305041339997842, + 2.1163640199993097, + 2.190599511999608, + 2.254729727999802, + 2.3131802959978813, + 2.3670923649988254, + 2.4182614739984274, + 2.4766606239973044, + 2.545225350997498, + 2.6055867269969895, + 2.659665966999455, + 2.715230447000067, + 2.78284747699945, + 2.842068581998319, + 2.9065254549968813, + 2.971242630999768, + 3.034353642997303, + 3.100675830999535, + 3.1558116969972616, + 3.214070089998131, + 3.28108055699704, + 3.3353142579981068, + 3.3943357549978828, + 3.453709694000281, + 3.5126013199987938, + 3.576584455997363, + 3.6519299789979414, + 3.7346794859986403, + 3.7965184679997037, + 3.8576476849993924, + 3.9170103859978553, + 3.9699302699991676, + 4.022215240998776, + 4.0794412169998395, + 4.138417523998214, + 4.197933327999635, + 4.267569140996784, + 4.324151043998427, + 4.388369251999393, + 4.475415230997896, + 4.55882640100026, + 4.6549523289977515, + 4.744847096997546, + 4.834320578996994, + 4.900272550999944, + 4.9819484669969825, + 5.0666834479998215, + 5.136139544996695, + 5.1886764880000555, + 5.2538797139968665, + 5.328271680999023, + 5.40297515400016, + 5.466780384998856, + 5.533192060996953, + 5.602427354999236, + 5.662636782999471, + 5.725061754998023, + 5.798684964996937, + 5.853141521998623, + 5.913959456996963, + 5.993807036997168, + 6.084860315997503, + 6.175643656999455, + 6.2531941429988365, + 6.316571707997355, + 6.3911962949969165, + 6.47940742299761, + 6.578149020999263, + 6.6573624929988, + 6.7197404479993565, + 6.794491027998447, + 6.860568324998894, + 6.920122186998924, + 6.985945334999997, + 7.047338550997665, + 7.1015015129996755, + 7.159093868998752, + 7.220071060997725, + 7.316406002999429 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.95197200294933, + "p95": 91.05327900033444, + "max": 320.8119949995307 + } + }, + { + "start": 30637.674009187, + "first": 30638.014745388, + "last": 30644.989894544, + "end": 30644.989971474, + "elapsed_s": 7.315962286997092, + "chunk_offsets_s": [ + 0.34073620100025437, + 0.40745707099995343, + 0.48646516199733014, + 0.5753562589998182, + 0.6448439919986413, + 0.6993544159995508, + 0.7578196340000432, + 0.8241955129997223, + 0.8880342619995645, + 0.9695545419999689, + 1.0395193320000544, + 1.1271023319968663, + 1.2096202269967762, + 1.284256346996699, + 1.3466866299968387, + 1.4124051149992738, + 1.4896178169983614, + 1.5664369629994326, + 1.6298011039980338, + 1.7079031939974811, + 1.7824492369982181, + 1.8510985289976816, + 1.9378403079972486, + 2.030490123997879, + 2.1160655329986184, + 2.190574166997976, + 2.254433177997271, + 2.3130296249983076, + 2.367020852998394, + 2.418267327997455, + 2.4766823099998874, + 2.5449729659994773, + 2.605835948997992, + 2.6592076089982584, + 2.7154307319979125, + 2.7828406849985186, + 2.8421573729974625, + 2.906577923997247, + 2.9711301229981473, + 3.03442380100023, + 3.1004550619982183, + 3.1559480919968337, + 3.2142573419987457, + 3.281133967997448, + 3.33544514699679, + 3.39445926499684, + 3.4539152269971964, + 3.5123006729991175, + 3.576560117999179, + 3.6512974119978026, + 3.734640964998107, + 3.796392176998779, + 3.857510123998509, + 3.9169158699987747, + 3.969717841999227, + 4.0221075669978745, + 4.078896084000007, + 4.138274056000228, + 4.197461720999854, + 4.267281380998611, + 4.3239363269967726, + 4.388661125998624, + 4.475137043998984, + 4.5587080489967775, + 4.654650849999598, + 4.744307361997926, + 4.8341792239989445, + 4.89996332499868, + 4.9818922729973565, + 5.066707947997202, + 5.136119511000288, + 5.18873466599689, + 5.2539047399986885, + 5.328379754999332, + 5.402910026998143, + 5.466746775997308, + 5.533520844997838, + 5.602631200999895, + 5.6625492139974085, + 5.725243942997622, + 5.798923530997854, + 5.8530365679980605, + 5.913432066998212, + 5.99377818299763, + 6.0848238409998885, + 6.175568443999509, + 6.253156450999086, + 6.316956994000066, + 6.391642407998006, + 6.4794674879995, + 6.578521837000153, + 6.657379209998908, + 6.720067242997175, + 6.794612224999582, + 6.860498073998315, + 6.919906637998793, + 6.985701436999079, + 7.0465073919986025, + 7.10137181099708, + 7.159399113999825, + 7.220562438997149, + 7.3158853569984785 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.79479900028673, + "p95": 90.74460299962084, + "max": 99.05434900065302 + } + }, + { + "start": 30637.673621707, + "first": 30637.843321608, + "last": 30644.989451032, + "end": 30644.989530684, + "elapsed_s": 7.315908977001527, + "chunk_offsets_s": [ + 0.16969990100187715, + 0.40837802199894213, + 0.48663153700181283, + 0.5752854099991964, + 0.6451103380022687, + 0.6999329210011638, + 0.7578650680006831, + 0.8247895300009986, + 0.8890299550002965, + 0.9695304849992681, + 1.0403561769999214, + 1.1274274610004795, + 1.209526539001672, + 1.2848494810023112, + 1.3469332080021559, + 1.4127375110001594, + 1.4898317379993387, + 1.5663981979996606, + 1.6301482920025592, + 1.7081951540021691, + 1.7826249310019193, + 1.8513466110016452, + 1.9388754710016656, + 2.030616480999015, + 2.1166161010005453, + 2.190776788000221, + 2.2549093900015578, + 2.3133499680006935, + 2.3678173660009634, + 2.4185138110005937, + 2.476932569999917, + 2.5454722290014615, + 2.605911291000666, + 2.6598465890019725, + 2.715864634999889, + 2.783011723000527, + 2.8423997099998815, + 2.9070040619990323, + 2.97136757699991, + 3.0347577030006505, + 3.10091947500041, + 3.1560585430015635, + 3.214497647000826, + 3.281327242999396, + 3.3355624649993842, + 3.3948795600008452, + 3.4542573090002406, + 3.5127472700005455, + 3.577278684999328, + 3.6517241110013856, + 3.7349514800007455, + 3.7969541890015535, + 3.8577953630010597, + 3.9173362300025474, + 3.9701711460002116, + 4.022464183999546, + 4.0796191500012355, + 4.138565026001743, + 4.1980527529995015, + 4.267712577999191, + 4.324194722001266, + 4.388751858001342, + 4.475576516000729, + 4.558986276999349, + 4.654823461001797, + 4.744980850002321, + 4.834452745999442, + 4.900437309999688, + 4.982371588001115, + 5.066772666999896, + 5.1364570789992285, + 5.189044286002172, + 5.254137523999816, + 5.328648988001078, + 5.403258960999665, + 5.467200431001402, + 5.533543371002452, + 5.6029540590025135, + 5.662975464001647, + 5.725325872001122, + 5.798953691999486, + 5.853255089001323, + 5.913345230001141, + 5.99407089899978, + 6.085264434001147, + 6.1762203329999465, + 6.253598004001105, + 6.317489323999325, + 6.391781615002401, + 6.479794843999116, + 6.578321111999685, + 6.657451650000439, + 6.720152328001859, + 6.794955173001654, + 6.860929878999741, + 6.920673507000174, + 6.986413450002146, + 7.047611584999686, + 7.102179115001491, + 7.159378121999907, + 7.220562639999116, + 7.315829325001687 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 65.9747059980873, + "p95": 91.1935350013664, + "max": 238.67812099706498 + } + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "chat", + "temperature": 0.0, + "streams": 5, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.1, + "per_stream_tps": 33.6, + "ttft_s_max": 0.42, + "request": { + "ttft_s": 0.254, + "decode_tps": 33.555, + "prefill_tps": null, + "itl_mean_ms": 29.802, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.517, + "round_gap_p50_ms": 69.523, + "round_gap_p95_ms": 100.111, + "tokens_per_chunk": 2.51 + }, + "pooled": { + "aggregate_prefill_tps": null, + "itl_mean_ms": 29.8, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.265, + "round_gap_p50_ms": 69.645, + "round_gap_p95_ms": 100.509, + "tokens_per_chunk": 2.51 + }, + "timing_valid": true, + "end_to_end_tps": 162.92195705446557, + "makespan_s": 7.85652236900205, + "steady_tps": 170.4, + "alone": { + "equal": 15, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 163.78644703309746, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.869895630999963, + "end_to_end_tps": 162.64510484205252, + "timing_valid": true, + "steady_tps": 170.03486106295054, + "per_stream_tps": [ + 33.1, + 33.5, + 33.9, + 34.2, + 32.8 + ], + "ttft_s": [ + 0.17, + 0.25, + 0.34, + 0.42, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.16829932299879147, + "decode_tps": 33.11566681283596, + "prefill_tps": null, + "itl_mean_ms": 30.19718750197082, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.51838699856307, + "round_gap_p50_ms": 69.4929460005369, + "round_gap_p95_ms": 100.50895299718832, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2535589559993241, + "decode_tps": 33.48871746277086, + "prefill_tps": null, + "itl_mean_ms": 29.860803152933286, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.22946700174361, + "round_gap_p50_ms": 70.01741400017636, + "round_gap_p95_ms": 100.11106700039818, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.3367868589994032, + "decode_tps": 33.860988461433514, + "prefill_tps": null, + "itl_mean_ms": 29.532510580397417, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.51672199752647, + "round_gap_p50_ms": 69.52265400104807, + "round_gap_p95_ms": 100.90157199738314, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.42086104600093677, + "decode_tps": 34.23854229806115, + "prefill_tps": null, + "itl_mean_ms": 29.20685090196225, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.67944000000716, + "round_gap_p50_ms": 69.3833179975627, + "round_gap_p95_ms": 98.3392569978605, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08525412699964363, + "decode_tps": 32.75728940661949, + "prefill_tps": null, + "itl_mean_ms": 30.527556403916712, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.4484860006487, + "round_gap_p50_ms": 69.64531099947635, + "round_gap_p95_ms": 100.16875700239325, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.864981708236098, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.26496599876555, + "round_gap_p50_ms": 69.64531099947635, + "round_gap_p95_ms": 100.50895299718832, + "round_gap_mean_ms": 75.40168649109114, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30644.993271361, + "first": 30645.161570684, + "last": 30652.861853497, + "end": 30652.861909855, + "elapsed_s": 7.868638494001061, + "chunk_offsets_s": [ + 0.16829932299879147, + 0.4967369170008169, + 0.583154480002122, + 0.6773806960009097, + 0.7484768780013837, + 0.8052748810005141, + 0.8643022710020887, + 0.9306524099993112, + 0.9970412850016146, + 1.0807263700007752, + 1.1576525550008228, + 1.2508009369994397, + 1.3378211800009012, + 1.4173115090015926, + 1.4897756050013413, + 1.5586550410007476, + 1.6417582040012348, + 1.7237844890005363, + 1.7910981429995445, + 1.8715858220020891, + 1.948269425000035, + 2.015370447999885, + 2.108780801001558, + 2.209289753998746, + 2.300023356001475, + 2.3784043380001094, + 2.4504559359993436, + 2.514528119001625, + 2.571706783000991, + 2.6258812730011414, + 2.6869184999995923, + 2.760145858999749, + 2.825717831001384, + 2.883343801000592, + 2.9431944860007206, + 3.009146766999038, + 3.0706820000013977, + 3.1386972299987974, + 3.21041086400146, + 3.2796203200014133, + 3.352614583000104, + 3.4101472340007604, + 3.476989048998803, + 3.552309671002149, + 3.609089566001785, + 3.6701703159997123, + 3.736016902999836, + 3.7974739499986754, + 3.8655894459989213, + 3.9442685579997487, + 4.036030609000591, + 4.095553080998798, + 4.1577738449996104, + 4.218796438999561, + 4.272954099000344, + 4.328137712000171, + 4.388038680001046, + 4.4574845620009, + 4.520540271001664, + 4.597234592998575, + 4.658003639000526, + 4.7240448180018575, + 4.815311346999806, + 4.904115307999746, + 5.006521810999402, + 5.1043190990021685, + 5.196837486000732, + 5.263727043999097, + 5.3549951819986745, + 5.44637789600165, + 5.525619364001614, + 5.579231026000343, + 5.649063777000265, + 5.727300971000659, + 5.8074434800000745, + 5.875287111000944, + 5.9407609369991405, + 6.017930215999513, + 6.081791077998787, + 6.150526209999953, + 6.225750034998782, + 6.2805216160013515, + 6.341630868999346, + 6.4255948060017545, + 6.5299389739993785, + 6.628775010001846, + 6.715636984001321, + 6.783955929000513, + 6.865874140999949, + 6.959354362999875, + 7.066333720998955, + 7.152874499999598, + 7.220161530000041, + 7.2973049270003685, + 7.366797873000905, + 7.430351571001665, + 7.5086338440014515, + 7.5741700870021305, + 7.63071437599865, + 7.692431267001666, + 7.758669713999552, + 7.868582136001351 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.4929460005369, + "p95": 100.50895299718832, + "max": 328.43759400202543 + } + }, + { + "start": 30644.992534197, + "first": 30645.246093153, + "last": 30652.860597957, + "end": 30652.8609666, + "elapsed_s": 7.8684324030000425, + "chunk_offsets_s": [ + 0.2535589559993241, + 0.49736042999938945, + 0.5837911029993847, + 0.6779029149984126, + 0.7491566719982075, + 0.8057109839974146, + 0.8653250809984456, + 0.9305620869999984, + 0.9977266019996023, + 1.0817764409985102, + 1.158284425000602, + 1.2511698259986588, + 1.3381885479975608, + 1.4178547310002614, + 1.4902699099984602, + 1.559336259997508, + 1.6427609729980759, + 1.7244805469999847, + 1.791324583999085, + 1.8722372209995228, + 1.9487672039977042, + 2.016414914000052, + 2.1094575709976198, + 2.209568637998018, + 2.3011928839987377, + 2.3792414819981786, + 2.4511408059988753, + 2.51539082599993, + 2.572577156999614, + 2.62653741000031, + 2.687743493999733, + 2.761094138997578, + 2.826633679000224, + 2.8842325339974195, + 2.943364558999747, + 3.0098200639986317, + 3.0711563789991487, + 3.1391272229993774, + 3.2107868009989033, + 3.2805527029995574, + 3.353436903998954, + 3.411543256999721, + 3.477223309000692, + 3.5528144630006864, + 3.6096753849997185, + 3.6710718869981065, + 3.736700411998754, + 3.7980562960001407, + 3.866414870997687, + 3.945413446999737, + 4.036665330000687, + 4.096210419000272, + 4.158458810998127, + 4.219615989997692, + 4.273875741000666, + 4.328831769998942, + 4.388688686998648, + 4.45837391799796, + 4.5219746729999315, + 4.597842150000361, + 4.658694800000376, + 4.724983394997253, + 4.815775806000602, + 4.905362699999387, + 5.006699189998471, + 5.105393074998574, + 5.1976225420003175, + 5.264693918998091, + 5.355794307997712, + 5.446886645000632, + 5.526408374000312, + 5.57965993399921, + 5.6496968979990925, + 5.727858668000408, + 5.807588401999965, + 5.875763047999499, + 5.9415450649976265, + 6.019285697999294, + 6.082566832999873, + 6.150873213999148, + 6.226871052000206, + 6.281582592997438, + 6.342462823999085, + 6.4271296089973475, + 6.530306289998407, + 6.628814095998678, + 6.71610530199905, + 6.784382549998554, + 6.8662712309997005, + 6.9599578409979586, + 7.066723721000017, + 7.153984822998609, + 7.221344345998659, + 7.297923722999258, + 7.367941136999434, + 7.431283196998265, + 7.509578627999872, + 7.574857822997728, + 7.631398958998034, + 7.692876985998737, + 7.759340562999569, + 7.8680637599973124 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.01741400017636, + "p95": 100.11106700039818, + "max": 243.80147400006535 + } + }, + { + "start": 30644.993085437, + "first": 30645.329872296, + "last": 30652.860662494, + "end": 30652.861139244, + "elapsed_s": 7.868053807000251, + "chunk_offsets_s": [ + 0.3367868589994032, + 0.49642673000198556, + 0.5828970040020067, + 0.6774661589988682, + 0.7483917999998084, + 0.8055235850006284, + 0.8641712700009521, + 0.9307292129997222, + 0.9975455900021188, + 1.0816517419989395, + 1.1579720740010089, + 1.2510997270001099, + 1.33803807699951, + 1.4172292510011175, + 1.4898780690018611, + 1.5586480470010429, + 1.641490951002197, + 1.7236886079990654, + 1.7909966750012245, + 1.8720013509991986, + 1.9480789609988278, + 2.0158123879991763, + 2.108645593001711, + 2.209547164999094, + 2.300840818999859, + 2.379269373999705, + 2.450941654999042, + 2.515200174999336, + 2.572218769000756, + 2.6257346679994953, + 2.686841077000281, + 2.760725554999226, + 2.8262875670006906, + 2.8837571350013604, + 2.94322099800047, + 3.0094809149995854, + 3.069908151999698, + 3.1390872900010436, + 3.210123017001024, + 3.2797243839995645, + 3.3530072520006797, + 3.4106953289992816, + 3.4768415489998006, + 3.5520403439986694, + 3.609404186001484, + 3.6702636879999773, + 3.736267943000712, + 3.7972865000010643, + 3.865489488998719, + 3.944680497999798, + 4.037033535001683, + 4.095069522001722, + 4.158105062000686, + 4.218681676000415, + 4.273521464001533, + 4.328380843999184, + 4.388010032998864, + 4.457444309999119, + 4.521217601999524, + 4.597369103001256, + 4.657974255998852, + 4.724058268999215, + 4.815138482001203, + 4.904571184000815, + 5.006755050999345, + 5.10427390300174, + 5.196790624999267, + 5.2636810789990705, + 5.355325870001252, + 5.446721486001479, + 5.525340935000713, + 5.579365473000507, + 5.648726106999675, + 5.727874815998803, + 5.80725300999984, + 5.875889061000635, + 5.941333467999357, + 6.018507308999688, + 6.082262078001804, + 6.150430625999434, + 6.226170918998832, + 6.281302375999076, + 6.341061778999574, + 6.425669533000473, + 6.530292213999928, + 6.628013505000126, + 6.716072258001077, + 6.784334028001467, + 6.866485276001185, + 6.959307121000165, + 7.066448928999307, + 7.153263018000871, + 7.220872194000549, + 7.297710396000184, + 7.367233050001232, + 7.430361343002005, + 7.508290609999676, + 7.574246607000532, + 7.631253275001654, + 7.692194092000136, + 7.75932081800056, + 7.867577057000744 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.52265400104807, + "p95": 100.90157199738314, + "max": 159.63987100258237 + } + }, + { + "start": 30644.992813667, + "first": 30645.413674713, + "last": 30652.861421693, + "end": 30652.861510496, + "elapsed_s": 7.868696829002147, + "chunk_offsets_s": [ + 0.42086104600093677, + 0.49657596399993054, + 0.5837247210001806, + 0.6775412800016056, + 0.7485483020027459, + 0.8056163159999414, + 0.8648630190000404, + 0.9309192190012254, + 0.9973889470020367, + 1.081433534000098, + 1.158071400001063, + 1.250946861000557, + 1.3383938370025135, + 1.41793859000245, + 1.490486212001997, + 1.559174026002438, + 1.6422662879995187, + 1.7246852110001782, + 1.7914990600002056, + 1.8723230300020077, + 1.948806833002891, + 2.0159099539996532, + 2.109358355002769, + 2.2097146500018425, + 2.3011671570020553, + 2.378914999000699, + 2.451262102000328, + 2.515240756001731, + 2.572448617000191, + 2.6263749030003964, + 2.6876207160021295, + 2.760892207003053, + 2.8261344350030413, + 2.8840747000031115, + 2.943019717000425, + 3.009686512999906, + 3.0707242520002183, + 3.139549542000168, + 3.211532507000811, + 3.2802301590018033, + 3.3529574120002508, + 3.4107312699998147, + 3.477011532002507, + 3.5528100070005166, + 3.609238360000745, + 3.6708339220022026, + 3.7363684240008297, + 3.798016800003097, + 3.865975957000046, + 3.944847696002398, + 4.036527136002405, + 4.095756763999816, + 4.158289243001491, + 4.219112345002941, + 4.273355753000942, + 4.328725749001023, + 4.38856689900058, + 4.457508392002637, + 4.521154735000891, + 4.597769487001642, + 4.65867372700086, + 4.724746984000376, + 4.815710441002011, + 4.904633539001225, + 5.007066869002301, + 5.105374050002865, + 5.197215339001559, + 5.264137133999611, + 5.355808128002536, + 5.446880569001223, + 5.5258785110017925, + 5.579729473000043, + 5.649458661002427, + 5.727419499002281, + 5.8077627980019315, + 5.876029385999573, + 5.941411046002031, + 6.0180927330002305, + 6.082127426001534, + 6.150414245999855, + 6.2259706870027, + 6.281029267000122, + 6.341945320000377, + 6.426266812002723, + 6.5302609960017435, + 6.628600252999604, + 6.7158961960012675, + 6.784154205000959, + 6.866048232001049, + 6.959725174001505, + 7.067429465001624, + 7.153389769002388, + 7.220887733001291, + 7.297920733002684, + 7.3673040510002465, + 7.431149850002839, + 7.508788295002887, + 7.574676985001133, + 7.631421115002013, + 7.692843119002646, + 7.759649203002482, + 7.868608026001311 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.3833179975627, + "p95": 98.3392569978605, + "max": 108.95882299882942 + } + }, + { + "start": 30644.99242101, + "first": 30645.077675137, + "last": 30652.86220202, + "end": 30652.862316641, + "elapsed_s": 7.869895630999963, + "chunk_offsets_s": [ + 0.08525412699964363, + 0.4976367620001838, + 0.5840577029994165, + 0.6785594670000137, + 0.749114076999831, + 0.8059165960003156, + 0.8647700360015733, + 0.9308907769991492, + 0.9980673620011657, + 1.0820734049993916, + 1.1589188990001276, + 1.2511838649988931, + 1.338165883000329, + 1.4178091440007847, + 1.490076129000954, + 1.5593759750008758, + 1.64230962199872, + 1.7248037149984157, + 1.79116627500116, + 1.8728194130017073, + 1.949050225000974, + 2.0164261530007934, + 2.1093747839986463, + 2.2095435410010396, + 2.3007418570014124, + 2.379570203000185, + 2.4511707560013747, + 2.515922883001622, + 2.5727777579995745, + 2.626339613001619, + 2.687920517000748, + 2.761514691999764, + 2.8274486720001732, + 2.8840741970016097, + 2.9436454979986593, + 3.0101898729990353, + 3.070633085000736, + 3.140096873001312, + 3.2107421129985596, + 3.2803344999992987, + 3.353630958001304, + 3.411419798001589, + 3.4774687610006367, + 3.5524750879994826, + 3.609997094001301, + 3.6711127540002053, + 3.7369661449993146, + 3.7978681240019796, + 3.8660952989994257, + 3.9449590500007616, + 4.037536292998993, + 4.095782978001807, + 4.1584430469993094, + 4.219300403001398, + 4.274346918999072, + 4.328840753001714, + 4.38861664399883, + 4.458026494001388, + 4.521649243000866, + 4.597460503999173, + 4.659383237998554, + 4.724979172999156, + 4.815744388000894, + 4.905318350000016, + 5.0070356219985115, + 5.104869596998469, + 5.197318082999118, + 5.2642082490019675, + 5.356044512001972, + 5.447135505000915, + 5.526133527000638, + 5.5796371920005186, + 5.649749355001404, + 5.728426956000476, + 5.807979959001386, + 5.875764284999605, + 5.941718564001349, + 6.019501246999425, + 6.083015822001471, + 6.151445276998857, + 6.226790287000767, + 6.282012486000895, + 6.341798059998837, + 6.427153336000629, + 6.53119597100158, + 6.628755500001716, + 6.716566855000565, + 6.784940574998473, + 6.8669361130014295, + 6.960166202999972, + 7.06655074399896, + 7.154158561999793, + 7.221621086999221, + 7.298211973000434, + 7.367857283999911, + 7.430904649001604, + 7.509412613999302, + 7.57480505499916, + 7.632139470999391, + 7.692730531998677, + 7.759948637998605, + 7.869781009998405 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.64531099947635, + "p95": 100.16875700239325, + "max": 412.3826350005402 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.17449812846672, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.853150495000591, + "end_to_end_tps": 162.99191016584533, + "timing_valid": true, + "steady_tps": 170.36113761965967, + "per_stream_tps": [ + 33.9, + 32.8, + 33.2, + 34.3, + 33.6 + ], + "ttft_s": [ + 0.34, + 0.09, + 0.17, + 0.42, + 0.25 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.335887889003061, + "decode_tps": 33.930401526092155, + "prefill_tps": null, + "itl_mean_ms": 29.47209449410758, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.16393700262415, + "round_gap_p50_ms": 68.26420499783126, + "round_gap_p95_ms": 100.33287099940935, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.08657641700119711, + "decode_tps": 32.8402903773532, + "prefill_tps": null, + "itl_mean_ms": 30.450400666664148, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.51931900143973, + "round_gap_p50_ms": 68.34396599879256, + "round_gap_p95_ms": 100.29740200116066, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.16999266399943735, + "decode_tps": 33.193540041160624, + "prefill_tps": null, + "itl_mean_ms": 30.12634382352653, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.00380099719041, + "round_gap_p50_ms": 68.27285699910135, + "round_gap_p95_ms": 100.55432899753214, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.4183531650014629, + "decode_tps": 34.30690196083184, + "prefill_tps": null, + "itl_mean_ms": 29.148653560782, + "itl_p50_ms": 0.0, + "itl_p95_ms": 91.48627799731912, + "round_gap_p50_ms": 68.3165039990854, + "round_gap_p95_ms": 98.5747109989461, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.2532515040002181, + "decode_tps": 33.554576172591226, + "prefill_tps": null, + "itl_mean_ms": 29.802194337261266, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.15128700088826, + "round_gap_p50_ms": 68.53342200338375, + "round_gap_p95_ms": 100.89719300231081, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.799937376468307, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.00380099719041, + "round_gap_p50_ms": 68.38329899983364, + "round_gap_p95_ms": 100.55432899753214, + "round_gap_mean_ms": 75.23746565345958, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30652.864952619, + "first": 30653.200840508, + "last": 30660.716224604, + "end": 30660.717188041, + "elapsed_s": 7.852235422000376, + "chunk_offsets_s": [ + 0.335887889003061, + 0.4895783330030099, + 0.5751600500007044, + 0.6758278880006401, + 0.7468859010004962, + 0.8052061040034459, + 0.8621787790034432, + 0.9293310360008036, + 0.9971566110034473, + 1.0812918060000811, + 1.158608740002819, + 1.2512927160023537, + 1.3377613880002173, + 1.4173611380028888, + 1.4836520630015002, + 1.550681598000665, + 1.6395494030002737, + 1.7229628590030188, + 1.7906858820024354, + 1.8723815590019512, + 1.9489297510008328, + 2.017091215002438, + 2.1101789050007937, + 2.2090394710030523, + 2.2992920150027203, + 2.3776333720015828, + 2.4463611020000826, + 2.5082978999998886, + 2.5650076170022658, + 2.6245517110000947, + 2.6860785900025803, + 2.76001483300206, + 2.8264065390030737, + 2.8825203140004305, + 2.9431066360011755, + 3.0113203190012428, + 3.0714463050026097, + 3.1388336080017325, + 3.209593464001955, + 3.2777859540001373, + 3.349285772001167, + 3.4082226480022655, + 3.469387415003439, + 3.5414036540023517, + 3.6033699110012094, + 3.667139784000028, + 3.731453386000794, + 3.793622712000797, + 3.8618153630013694, + 3.9413883830020495, + 4.032985071000439, + 4.091888448001555, + 4.152984860000288, + 4.2140331260015955, + 4.267573299002834, + 4.323338374000741, + 4.382755288002954, + 4.447255559000041, + 4.50782450800034, + 4.588519888002338, + 4.65114175400231, + 4.718523766001454, + 4.812276930002554, + 4.901190079002845, + 5.004171764001512, + 5.099483298003179, + 5.1900674110002, + 5.2576737480012525, + 5.347303794002073, + 5.437419028003205, + 5.5129230370002915, + 5.568968932002463, + 5.6430631690018345, + 5.721373678003147, + 5.800821778000682, + 5.868881260001217, + 5.9362091530019825, + 6.014924442002666, + 6.079498622002575, + 6.147762827000406, + 6.221539472000586, + 6.277141829999891, + 6.336895649001235, + 6.424673560002702, + 6.520230678001099, + 6.620563549000508, + 6.708577659002913, + 6.775062907003303, + 6.856390547000046, + 6.9485544840026705, + 7.054758487000072, + 7.138004252003157, + 7.2050029290003295, + 7.280227082002966, + 7.350313198003278, + 7.414388512002915, + 7.486858051001036, + 7.550887156001409, + 7.613547109001956, + 7.676304393000464, + 7.742005448002601, + 7.851271985000494 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.26420499783126, + "p95": 100.33287099940935, + "max": 153.6904439999489 + } + }, + { + "start": 30652.864456534, + "first": 30652.951032951, + "last": 30660.715885121, + "end": 30660.716508115, + "elapsed_s": 7.852051581001433, + "chunk_offsets_s": [ + 0.08657641700119711, + 0.4896477039983438, + 0.5761336049981765, + 0.6764310069993371, + 0.7480685470000026, + 0.805341661998682, + 0.8627955999982078, + 0.9294527979982377, + 0.9970993269998871, + 1.0815249560000666, + 1.1585045419997186, + 1.2510238610011584, + 1.3376634950000152, + 1.417752509998536, + 1.4839237720007077, + 1.5512887820004835, + 1.639512798999931, + 1.7236492109987012, + 1.7906248520012014, + 1.8724078759987606, + 1.9489141820013174, + 2.017113532001531, + 2.110626042998774, + 2.209241201999248, + 2.299187371998414, + 2.3779066509996483, + 2.446250616998441, + 2.508939000999817, + 2.565642042998661, + 2.6252544679991843, + 2.687064112000371, + 2.760256980000122, + 2.826638793998427, + 2.8827205240013427, + 2.9432469210005365, + 3.0115316869996605, + 3.071623275998718, + 3.1394732519984245, + 3.2095745359983994, + 3.2775920319982106, + 3.349480590000894, + 3.4086561969998, + 3.4695811919991684, + 3.5420145019998017, + 3.6040021599983447, + 3.667500330000621, + 3.7318634379989817, + 3.7945722980002756, + 3.8620299479989626, + 3.9414176370009955, + 4.032894310999836, + 4.0922619750017475, + 4.153223008001078, + 4.214424050998787, + 4.268397059000563, + 4.323453034998238, + 4.3833088829997, + 4.447655687999941, + 4.508136955999362, + 4.589093121998303, + 4.651538089998212, + 4.718800937000196, + 4.812420499001746, + 4.901198235998891, + 5.004061191000801, + 5.099465111998143, + 5.1905200709989, + 5.257824616001017, + 5.347264591000567, + 5.437838459998602, + 5.513104301000567, + 5.569131197000388, + 5.64291917599985, + 5.721913395998854, + 5.800880757000414, + 5.868243868000718, + 5.936097190999135, + 6.01535764000073, + 6.079394037999009, + 6.147887671999342, + 6.222267402001307, + 6.276944684999762, + 6.337305173998175, + 6.423964353998599, + 6.520496389999607, + 6.621144880999054, + 6.7087521700013895, + 6.775766989001568, + 6.8564879420009675, + 6.949233051000192, + 7.05484258299839, + 7.138182077000238, + 7.205313313999795, + 7.280366621998837, + 7.350467494998156, + 7.414562943999044, + 7.487033828001586, + 7.551060580000922, + 7.613985252999555, + 7.676850031999493, + 7.742308806999063, + 7.851428587000555 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.34396599879256, + "p95": 100.29740200116066, + "max": 403.0712869971467 + } + }, + { + "start": 30652.864541912, + "first": 30653.034534576, + "last": 30660.716752251, + "end": 30660.716831112, + "elapsed_s": 7.852289200000087, + "chunk_offsets_s": [ + 0.16999266399943735, + 0.490157554999314, + 0.5762267300015083, + 0.676833040000929, + 0.7483051529998193, + 0.8056777480014716, + 0.8629349359980552, + 0.9300529370011645, + 0.9972242069998174, + 1.0819352610014903, + 1.1588539260010293, + 1.2516107310002553, + 1.3379921839987219, + 1.4178476509987377, + 1.4835355389986944, + 1.5515264429996023, + 1.6398249529993336, + 1.7228964189998806, + 1.791135243998724, + 1.8726379979998455, + 1.9493720549980935, + 2.0177722820008057, + 2.110230928999954, + 2.209532897999452, + 2.299317549000989, + 2.3783438589998696, + 2.446575412999664, + 2.508397157998843, + 2.5658756849989004, + 2.624736235000455, + 2.6865500079984486, + 2.7604751590006344, + 2.82676732300024, + 2.8829951570005505, + 2.9434987120002916, + 3.0116353900011745, + 3.071891040999617, + 3.139434579999943, + 3.2100869540008716, + 3.27784843200061, + 3.3495045819981897, + 3.408714297998813, + 3.4697059099999024, + 3.5420840000006137, + 3.6043288110013236, + 3.6673577619985736, + 3.731592340998759, + 3.794409338999685, + 3.8626821959987865, + 3.9411297340011515, + 4.033133534998342, + 4.0923970000003465, + 4.153071316999558, + 4.214499821999198, + 4.268171004001488, + 4.323677213000337, + 4.38352238600055, + 4.447437891998561, + 4.508575822997955, + 4.5887955840007635, + 4.65174037199904, + 4.719547455999418, + 4.813061185999686, + 4.901956424000673, + 5.004516829998465, + 5.099832206000428, + 5.190379552997911, + 5.257866934000049, + 5.347350220999942, + 5.43812786199851, + 5.513539180999942, + 5.569341400998383, + 5.643338385998504, + 5.7216740240000945, + 5.801405382000667, + 5.869520052001462, + 5.936572627000714, + 6.014961909000704, + 6.079772558998229, + 6.147958478999499, + 6.2221182089997455, + 6.277209691997996, + 6.337347779000993, + 6.425279153998417, + 6.52077335300055, + 6.621327681998082, + 6.708865536998928, + 6.775293595997937, + 6.856947723998019, + 6.948916630000895, + 7.055213658000866, + 7.138361034998525, + 7.205054704998474, + 7.280392820000998, + 7.350571048998972, + 7.415147824998712, + 7.487104169998929, + 7.551082280999253, + 7.614031970999349, + 7.6774367859979975, + 7.742875056999765, + 7.852210338998702 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.27285699910135, + "p95": 100.55432899753214, + "max": 320.16489099987666 + } + }, + { + "start": 30652.864871099, + "first": 30653.283224264, + "last": 30660.716130922, + "end": 30660.716265483, + "elapsed_s": 7.851394384000741, + "chunk_offsets_s": [ + 0.4183531650014629, + 0.4899488000010024, + 0.5757770790005452, + 0.6761020770027244, + 0.7470585130031395, + 0.8049717070025508, + 0.8624604750002618, + 0.9290845230025297, + 0.9974010270016151, + 1.0812574410010711, + 1.1584406260008109, + 1.2508540000017092, + 1.337514747003297, + 1.4171083990004263, + 1.483851389002666, + 1.5511223310022615, + 1.6394315490033478, + 1.7224812940003176, + 1.7909747470002912, + 1.8722424330007925, + 1.9487163730009343, + 2.016877308000403, + 2.1100403720010945, + 2.2086150830000406, + 2.2992488630006847, + 2.377937250003015, + 2.4457789570005843, + 2.5086103410030773, + 2.5654445700020005, + 2.6245020770002156, + 2.6866997750003065, + 2.75992145300188, + 2.8262823470031435, + 2.882366126003035, + 2.942899710000347, + 3.0112101020022237, + 3.071280131000094, + 3.1389767680011573, + 3.209285203000036, + 3.2773538700021163, + 3.3490258180027013, + 3.408206097999937, + 3.4692123570021067, + 3.541675774002215, + 3.6035307410020323, + 3.6671628820004116, + 3.731376077001187, + 3.7942099450010573, + 3.8615600500015717, + 3.941151239003375, + 4.032637517000694, + 4.0918071390005935, + 4.152859102003276, + 4.214231090001704, + 4.267950978002773, + 4.3231602600026235, + 4.382947810001497, + 4.447290037001949, + 4.5081074480003736, + 4.588736387002427, + 4.651171415000135, + 4.719052767002722, + 4.811953946002177, + 4.900930911000614, + 5.003926666002371, + 5.0992667870013975, + 5.189985350003553, + 5.257608573003381, + 5.347551775001193, + 5.437850582002284, + 5.512889026002085, + 5.568609312002081, + 5.642441164000047, + 5.721703291001177, + 5.800708282000414, + 5.86800648800272, + 5.935562678001588, + 6.014512661000481, + 6.07914009200249, + 6.1471775750032975, + 6.221491568001511, + 6.276748280000902, + 6.337202355000045, + 6.423829459999979, + 6.519863745001203, + 6.621476238000469, + 6.708724227002676, + 6.775458654999966, + 6.856685121001647, + 6.9489542310002435, + 7.0547222690001945, + 7.137813801000448, + 7.205042785000842, + 7.280503633002809, + 7.350644809001096, + 7.4143970610020915, + 7.486898884002585, + 7.550517601001047, + 7.613799283000844, + 7.676704204000998, + 7.742427267003222, + 7.851259823000873 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.3165039990854, + "p95": 98.5747109989461, + "max": 108.83255599765107 + } + }, + { + "start": 30652.864348875, + "first": 30653.117600379, + "last": 30660.717159935, + "end": 30660.71749937, + "elapsed_s": 7.853150495000591, + "chunk_offsets_s": [ + 0.2532515040002181, + 0.48955221000142046, + 0.5758110290007608, + 0.6767082220030716, + 0.7480771910013573, + 0.8058962510003767, + 0.8633226580022892, + 0.9303868770002737, + 0.9978729909998947, + 1.0819863060023636, + 1.159337189001235, + 1.2514884760021232, + 1.3386799110012362, + 1.4180723960016621, + 1.4846151800011285, + 1.5519333400006872, + 1.6402629820004222, + 1.7232421080007043, + 1.790861618002964, + 1.8728844320030476, + 1.949483524000243, + 2.0178820230030396, + 2.111011185999814, + 2.2095377660007216, + 2.299591049002629, + 2.3783051740028895, + 2.446519650002301, + 2.508639142000902, + 2.5661197660010657, + 2.6252378070021223, + 2.6869352480025555, + 2.7605692300021474, + 2.827233698000782, + 2.883048076000705, + 2.943595841003116, + 3.01197914000295, + 3.0721290390029026, + 3.1393458929996996, + 3.2103378050014726, + 3.2785211880000134, + 3.35005313000147, + 3.4085952320019715, + 3.46995508100008, + 3.5418847910004843, + 3.6043158330030565, + 3.6680025550012942, + 3.7321525100014696, + 3.794383571999788, + 3.8631338760023937, + 3.9415633420030645, + 4.033357992000674, + 4.0926223210008175, + 4.15319151100266, + 4.214585970999906, + 4.268399430002319, + 4.323972001002403, + 4.3832809230007115, + 4.447575772002892, + 4.508485970000038, + 4.58908268900268, + 4.651508992999879, + 4.720042415003263, + 4.812975606000691, + 4.902005567000742, + 5.00495853600296, + 5.10017721999975, + 5.19042232600259, + 5.258209482002712, + 5.347865986001125, + 5.4382827560002625, + 5.513569021000876, + 5.569371176999994, + 5.643229407000035, + 5.721931117001077, + 5.801458191002894, + 5.868879483001365, + 5.936461471999792, + 6.015631120000762, + 6.080262825002137, + 6.148069054001098, + 6.222255705000862, + 6.277703286999895, + 6.337783640999987, + 6.424791509001807, + 6.520767581001564, + 6.621738719000859, + 6.709022256000026, + 6.776071920001414, + 6.856852062999678, + 6.949009613999806, + 7.055173603002913, + 7.13877501600291, + 7.205350580003142, + 7.28113083900098, + 7.350954299003206, + 7.415046899001027, + 7.487335877001897, + 7.551316725002835, + 7.614280900001177, + 7.677330818001792, + 7.742999205001979, + 7.852811060001841 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 68.53342200338375, + "p95": 100.89719300231081, + "max": 236.30070600120234 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 164.11689285754832, + "requested_requests": 5, + "timed_requests": 5, + "makespan_s": 7.85652236900205, + "end_to_end_tps": 162.92195705446557, + "timing_valid": true, + "steady_tps": 170.42711844073224, + "per_stream_tps": [ + 32.8, + 33.9, + 33.6, + 33.2, + 34.3 + ], + "ttft_s": [ + 0.09, + 0.34, + 0.26, + 0.17, + 0.42 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 31, + "ttft_s": 0.08735474399873056, + "decode_tps": 32.83106915735957, + "prefill_tps": null, + "itl_mean_ms": 30.45895323137337, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.59429399773944, + "round_gap_p50_ms": 70.35573199755163, + "round_gap_p95_ms": 97.30553199915448, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.33978782299891463, + "decode_tps": 33.93435110658346, + "prefill_tps": null, + "itl_mean_ms": 29.46866427058315, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.56169999955455, + "round_gap_p50_ms": 70.02920100057963, + "round_gap_p95_ms": 97.1824980006204, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.25702111999999033, + "decode_tps": 33.557365410520696, + "prefill_tps": null, + "itl_mean_ms": 29.79971722352453, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.35131699845078, + "round_gap_p50_ms": 69.90615000177058, + "round_gap_p95_ms": 97.15338499881909, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.17369772199890576, + "decode_tps": 33.1980400847313, + "prefill_tps": null, + "itl_mean_ms": 30.1222601529398, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.32158499819343, + "round_gap_p50_ms": 70.24042799821473, + "round_gap_p95_ms": 97.17158199782716, + "tokens_per_chunk": 2.5098039215686274 + }, + { + "prompt_tokens": 31, + "ttft_s": 0.42480633100058185, + "decode_tps": 34.3190821328676, + "prefill_tps": null, + "itl_mean_ms": 29.138308423531342, + "itl_p50_ms": 0.0, + "itl_p95_ms": 92.98936000050162, + "round_gap_p50_ms": 69.89645499925246, + "round_gap_p95_ms": 97.20858699802193, + "tokens_per_chunk": 2.5098039215686274 + } + ], + "pooled": { + "streams_timed": 5, + "itl_mean_ms": 29.79758066039044, + "itl_p50_ms": 0.0, + "itl_p95_ms": 93.35131699845078, + "round_gap_p50_ms": 70.15276499805623, + "round_gap_p95_ms": 97.27267299967934, + "round_gap_mean_ms": 75.23151552870854, + "tokens_per_chunk": 2.5098039215686274, + "aggregate_prefill_tps": null + }, + "evidence": [ + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + }, + { + "prompt_tokens": 31, + "cached_tokens": 24, + "tokens": 256, + "cache_state": "reused", + "complete": true, + "token_sha": "9524a6a9f64e", + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7" + } + ], + "client_intervals": [ + { + "start": 30660.719166822, + "first": 30660.806521566, + "last": 30668.57355464, + "end": 30668.573634298, + "elapsed_s": 7.854467476001446, + "chunk_offsets_s": [ + 0.08735474399873056, + 0.4937574190007581, + 0.5774980089991004, + 0.6715749209979549, + 0.7462423440010753, + 0.8064218789986626, + 0.86373149100109, + 0.9321897160007211, + 0.9984946260010474, + 1.0829449880002358, + 1.1603065409981355, + 1.2545346000006248, + 1.3390945850005664, + 1.4185556000011275, + 1.484907613001269, + 1.5550603779993253, + 1.6380415969979367, + 1.725117048001266, + 1.7954727799988177, + 1.8770651250015362, + 1.9535078250009974, + 2.0196383520014933, + 2.1132326459992328, + 2.213576041998749, + 2.303342718998465, + 2.3809047030008514, + 2.448388045999309, + 2.51162801999817, + 2.5689071439992404, + 2.623676622999483, + 2.6831653189983626, + 2.7625141979988257, + 2.8297353699999803, + 2.8856692879999173, + 2.946050006998121, + 3.013681082000403, + 3.075016618000518, + 3.143947656000819, + 3.214518108001357, + 3.2813828979997197, + 3.3523960310012626, + 3.4115409400001226, + 3.471827811001276, + 3.5443663180012663, + 3.6032526419985516, + 3.66215794800155, + 3.7301924849998613, + 3.7959219379990827, + 3.865242535001016, + 3.943633269998827, + 4.036798017001274, + 4.095310278000397, + 4.1569879119997495, + 4.220561408998037, + 4.274864126000466, + 4.3309849690012925, + 4.391582121999818, + 4.455767234998348, + 4.518256793999171, + 4.593766410998796, + 4.6522651209998, + 4.721070785999473, + 4.817937400999654, + 4.90800143999877, + 5.011064505000832, + 5.1054662739989, + 5.200097281998751, + 5.26687337399926, + 5.3545782859982864, + 5.444720404000691, + 5.520294231999287, + 5.57588246999876, + 5.643488154000806, + 5.72471773000143, + 5.809164375001274, + 5.8764296879999165, + 5.944961893001164, + 6.023617639999429, + 6.087180015998456, + 6.15486648300066, + 6.229848135000793, + 6.285644764000608, + 6.346497959999397, + 6.433942265000951, + 6.5312477970001055, + 6.626825936000387, + 6.713431865999155, + 6.785953210001026, + 6.867605998999352, + 6.9602942150013405, + 7.0654362159984885, + 7.150308469997981, + 7.218210570998053, + 7.290011206998315, + 7.360374088999379, + 7.4221864370010735, + 7.493710710001324, + 7.559388204997958, + 7.616077982998831, + 7.676727354999457, + 7.747696373000508, + 7.854387817998941 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.35573199755163, + "p95": 97.30553199915448, + "max": 406.40267500202754 + } + }, + { + "start": 30660.720164335, + "first": 30661.059952158, + "last": 30668.574461547, + "end": 30668.574566606, + "elapsed_s": 7.854402270997525, + "chunk_offsets_s": [ + 0.33978782299891463, + 0.49329976399894804, + 0.5769481279967295, + 0.6710812849996728, + 0.7449484689968813, + 0.8064054849965032, + 0.863204641998891, + 0.9316721339964715, + 0.9982070629994269, + 1.0826172159977432, + 1.1599208439984068, + 1.2537896399990132, + 1.3383706249987881, + 1.4174443829979282, + 1.4844387029988866, + 1.5544679039994662, + 1.6377442199991492, + 1.7245372029974533, + 1.7943942789970606, + 1.8762647209987335, + 1.9529822389995388, + 2.019289291998575, + 2.1127874859994336, + 2.213035663997289, + 2.302698113999213, + 2.380309377997037, + 2.4471149329983746, + 2.511260150997259, + 2.568241954999394, + 2.6230122829983884, + 2.6825919219991192, + 2.7619223949986917, + 2.8293558649966144, + 2.885258091999276, + 2.945582887998171, + 3.0128129749973596, + 3.0743220889999066, + 3.1432594979996793, + 3.2139392199969734, + 3.2811119869984395, + 3.351754704999621, + 3.4107493379997322, + 3.4705280489979486, + 3.5437599799988675, + 3.602686060999986, + 3.661523695998767, + 3.7293701199996576, + 3.7948886369995307, + 3.864110927999718, + 3.94281543499892, + 4.0363771349984745, + 4.0947927689994685, + 4.156668245999754, + 4.22009241199703, + 4.274415936997684, + 4.330588396998792, + 4.391128858998854, + 4.454603137997765, + 4.5174593979973, + 4.592946142998699, + 4.651851502996578, + 4.720326879996719, + 4.817087650997564, + 4.907906185999309, + 5.010218866998912, + 5.104863597996882, + 5.199383480998222, + 5.266006848996767, + 5.353779735996795, + 5.444361377998575, + 5.519061294999119, + 5.574931420997018, + 5.642697862997011, + 5.723650560998067, + 5.808380885999213, + 5.8760230939988105, + 5.944312960997195, + 6.022881606997544, + 6.086673466998036, + 6.154066939998302, + 6.229429711998819, + 6.285353406998183, + 6.346048820996657, + 6.433441925997613, + 6.530624423998233, + 6.6261364619967935, + 6.71258600299916, + 6.785111556997435, + 6.866932370998256, + 6.95967902099801, + 7.064544593999017, + 7.1497652669968375, + 7.217271493998851, + 7.289398428998538, + 7.359735573998478, + 7.421109969996905, + 7.492763740996452, + 7.558443221998459, + 7.6153373869965435, + 7.676240842996776, + 7.747302274998219, + 7.854297211997618 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.02920100057963, + "p95": 97.1824980006204, + "max": 153.5119410000334 + } + }, + { + "start": 30660.71942546, + "first": 30660.97644658, + "last": 30668.575374472, + "end": 30668.575487374, + "elapsed_s": 7.8560619140007475, + "chunk_offsets_s": [ + 0.25702111999999033, + 0.4941053180009476, + 0.5776282299993909, + 0.6718904730005306, + 0.7463905080003315, + 0.8068205800009309, + 0.8638367269995797, + 0.9326555129991903, + 0.9988466229988262, + 1.0833851249990403, + 1.1613054670015117, + 1.2546567839999625, + 1.3394808150005701, + 1.4182292100013, + 1.4851403969987587, + 1.5550465470005292, + 1.6384316529984062, + 1.725383844001044, + 1.7952758760002325, + 1.8771622130007017, + 1.9536530100012897, + 2.0197456030000467, + 2.11362120999911, + 2.2138992229993164, + 2.303745543998957, + 2.3812885929983167, + 2.448330502000317, + 2.5119701039984648, + 2.5693996959998913, + 2.624458941001649, + 2.683545240000967, + 2.762751860998833, + 2.830140723999648, + 2.8860590850017616, + 2.9461007569989306, + 3.0136858969999594, + 3.0751220249985636, + 3.14395370100101, + 3.2145567250008753, + 3.2815575929998886, + 3.3528808010014473, + 3.411853443001746, + 3.471807431000343, + 3.5445575789999566, + 3.603225011000177, + 3.6621441620009136, + 3.7300291280007514, + 3.7955401219987834, + 3.86470229599945, + 3.9439124659984373, + 4.036863666999125, + 4.09529497100084, + 4.157331654998416, + 4.220525246000761, + 4.275009738001245, + 4.3311858480010414, + 4.391805472001579, + 4.45542458500131, + 4.518475007000234, + 4.593631559000642, + 4.65239111000119, + 4.721019994998642, + 4.817783950999001, + 4.908609704998526, + 5.011407512000005, + 5.105948494001495, + 5.200058638001792, + 5.266468878999149, + 5.35506493900175, + 5.444737123001687, + 5.519736405000003, + 5.575489563001611, + 5.64349122099884, + 5.724599730998307, + 5.808768330000021, + 5.8768595379988255, + 5.9448139800006174, + 6.023853889000748, + 6.0873135090005235, + 6.155251899999712, + 6.230328000001464, + 6.286446223999519, + 6.347132803999557, + 6.434241317001579, + 6.531394702000398, + 6.6268535050003265, + 6.713263033998373, + 6.785235731000284, + 6.867943815999752, + 6.960623132999899, + 7.065946041999268, + 7.150701375001518, + 7.217544455001189, + 7.290253743001813, + 7.360609406001458, + 7.42146342200067, + 7.493603097998857, + 7.558932365998771, + 7.616461941001035, + 7.676856748999853, + 7.747681319000549, + 7.855949011998746 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.90615000177058, + "p95": 97.15338499881909, + "max": 237.08419800095726 + } + }, + { + "start": 30660.719086745, + "first": 30660.892784467, + "last": 30668.573960806, + "end": 30668.575053646, + "elapsed_s": 7.855966900999192, + "chunk_offsets_s": [ + 0.17369772199890576, + 0.49430331099938485, + 0.5784162579984695, + 0.6719459659980203, + 0.7462355259995093, + 0.8072309210001549, + 0.8641266560007352, + 0.9325986779986124, + 0.9990750100005243, + 1.0834522559998732, + 1.1606677249983477, + 1.2549124459983432, + 1.3395094999978028, + 1.4187632739995024, + 1.485434999998688, + 1.5553149009974732, + 1.638665275000676, + 1.725522696000553, + 1.7957631239987677, + 1.8772144579997985, + 1.95379499900082, + 2.0205607909992978, + 2.1136398130001908, + 2.2141714350000257, + 2.303834715999983, + 2.3818806769995717, + 2.4482902370000375, + 2.512141960000008, + 2.5694815530005144, + 2.6245322190006846, + 2.683417167998414, + 2.7633856239990564, + 2.830270885999198, + 2.8865758779975295, + 2.9466291869975976, + 3.01422533799996, + 3.0755145830007677, + 3.1442311629980395, + 3.214726933998463, + 3.281837168000493, + 3.352999152000848, + 3.412394883998786, + 3.4718561739973666, + 3.5447088699984306, + 3.6038656389973767, + 3.6625221709982725, + 3.730518263000704, + 3.7957829639999545, + 3.8648429909990227, + 3.943953030000557, + 4.037274614998751, + 4.095766210997681, + 4.157255092999549, + 4.220966973000031, + 4.27546330900077, + 4.3316373210000165, + 4.392237498999748, + 4.4559608710005705, + 4.519028643000638, + 4.5939409359998535, + 4.652425128999312, + 4.721266166998248, + 4.818229983000492, + 4.908720567000273, + 5.012014846000966, + 5.106074912997428, + 5.200724967999122, + 5.266668362997734, + 5.355487714998162, + 5.444990048999898, + 5.519841441000608, + 5.575886986000114, + 5.644270916000096, + 5.724608702999831, + 5.809003826998378, + 5.877344301999983, + 5.945518705997529, + 6.0242280079983175, + 6.087695645997883, + 6.1555064269996365, + 6.230952218000311, + 6.287017927999841, + 6.347376974998042, + 6.434482016000402, + 6.531653597998229, + 6.627087081000354, + 6.713338396999461, + 6.785773984000116, + 6.868208362997393, + 6.960676423997938, + 7.0659068069981, + 7.150757740000699, + 7.217951415997959, + 7.2901853949988435, + 7.360714251997706, + 7.421709497997654, + 7.493596642998455, + 7.559783505999803, + 7.616731688998698, + 7.677120927997748, + 7.7476863080009934, + 7.854874060998554 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 70.24042799821473, + "p95": 97.17158199782716, + "max": 320.6055890004791 + } + }, + { + "start": 30660.718965005, + "first": 30661.143771336, + "last": 30668.574039984, + "end": 30668.574241394, + "elapsed_s": 7.85527638900021, + "chunk_offsets_s": [ + 0.42480633100058185, + 0.4941815080019296, + 0.577829568002926, + 0.672632613001042, + 0.7469139929999074, + 0.8069039239999256, + 0.8641750580027292, + 0.9326258410001174, + 0.998954696002329, + 1.0834299930029374, + 1.160888012000214, + 1.2548665270005586, + 1.3395048760030477, + 1.4188306590003776, + 1.485348171001533, + 1.555553450001753, + 1.638572699001088, + 1.7255200560030062, + 1.795942772001581, + 1.8771639610022248, + 1.9543171050027013, + 2.020553333000862, + 2.1135426930013637, + 2.2138944110010925, + 2.3042698140016, + 2.381202985001437, + 2.4485001360008027, + 2.512073826001142, + 2.5692629030017997, + 2.62381661100153, + 2.683475413999986, + 2.762940606000484, + 2.830115441000089, + 2.886060387001635, + 2.9464648110006237, + 3.0144525860014255, + 3.075849452001421, + 3.14488423900184, + 3.2150881800007483, + 3.2822459030030586, + 3.353521062002983, + 3.412564080001175, + 3.471803678003198, + 3.544904571001098, + 3.6033490460031317, + 3.662302881002688, + 3.7302962680005294, + 3.7957733780021954, + 3.8655789110016485, + 3.943748546000279, + 4.037243422000756, + 4.095824650001305, + 4.1575759570005175, + 4.221193516001222, + 4.275358779003, + 4.331539258000703, + 4.391983911002171, + 4.456054121001216, + 4.519241406000219, + 4.593789047001337, + 4.65265820300192, + 4.721145344003162, + 4.818418017002841, + 4.908466018001491, + 5.011209950000193, + 5.105933044000267, + 5.200761895001051, + 5.267277531002037, + 5.354706894002447, + 5.445499344001291, + 5.520435180002096, + 5.5758694790019945, + 5.6443126930025755, + 5.724644781999814, + 5.8089472980027494, + 5.87703221600168, + 5.9455572180013405, + 6.024265990999993, + 6.08768271700319, + 6.155792654000834, + 6.2309559830027865, + 6.286727563001477, + 6.347127263001312, + 6.434496584002773, + 6.5317051710007945, + 6.627260823002871, + 6.714009104001889, + 6.785614398002508, + 6.8680093080001825, + 6.960610662001272, + 7.066349710003124, + 7.150409341000341, + 7.2181587209997815, + 7.290532442002586, + 7.360428897001839, + 7.422813514000154, + 7.493827122001676, + 7.559505738001462, + 7.616690288003156, + 7.6771556150015385, + 7.748697062001156, + 7.855074979001074 + ], + "chunk_gaps_ms": { + "count": 101, + "p50": 69.89645499925246, + "p95": 97.20858699802193, + "max": 106.37791699991794 + } + } + ], + "equal_alone": [ + true, + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9524a6a9f64e2e0afda40fdd9498494f9d1f7ee46d1429b2dfb4a669859beb4c", + "output_sha256": "926b51df69ad6e2afaca8f6e27bef38db16115da2509b66cb26d1897af4705d7", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/receipts/flashnext-cuda-sampled-heads/README.md b/docs/receipts/flashnext-cuda-sampled-heads/README.md new file mode 100644 index 0000000000..3899da6181 --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/README.md @@ -0,0 +1,75 @@ +# Flash Next CUDA: sampled streams in the batched MTP head pass + +A shared round's streams with a temperature above zero were left out of the batched MTP head pass and drafted alone, +one head forward per draft token (the temperature skip in `cuda_lanes_draft.zig`'s `batched`). With this change they +join the pass: each step, after the shared best-column download, a sampled lane's score row is downloaded and its +draft drawn on the host under the lane's own keyed rule at the drafted position, the draw its lone `pick` makes. +Greedy lanes keep their best column. `TF_FLASHNEXT_SAMPLED_HEADS=0` restores the previous behaviour. + +## Qualified scope + +TensorFold 1.1.8 (`03d08f1ef5e385f6ecccb49d54e31b44e75ed422`) plus this change (the commit in `commit.txt`), built +from source with Zig 0.17.0 and nvcc 13.3 (`zig build -Doptimize=fast fatbins native install`) in the +`nvcr.io/nvidia/pytorch:26.07-py3` image. One DGX Spark (GB10, sm_121, 128 GB unified memory), kernel driver +580.178.04, CUDA 13.3 forward-compatibility driver 610.43.02. Checkpoint `nvidia/Qwen3.8-Flash-Next-FP4-Local` at +`b9ce18a6887e18c089fdea8509a33e63429a3300`. + +Server: `tensorfold-native serve CHECKPOINT --context 131072 --parallel 5 --kv-cache bf16 --max-tokens 32768 +--temperature 1.0 --top-p 0.95 --top-k 20 --thinking`, `TENSORFOLD_MEMORY_RESERVE_GIB=9`, no vision, no other +`TF_FLASHNEXT_*` variables. The server reported: model 73.28 GiB; 5 streams at once, 4.05 GiB each at a +131072-token window, of 22.4 GiB left after a 9.0 GiB reserve; a 2.1 GiB prompt cache; own glue kernels. + +## Correctness + +- `tf-cuda-test flashnext-mtp-parity CHECKPOINT zig/tests/cuda/flashnext/lr7/mid.ids` ([parity.txt](parity.txt)): + 18 shapes pass. The six greedy shapes of 1.1.8 (confidence 0 and 2), and twelve new ones: every stream sampled + (seed 19 + i, position 5 + 3i, temperature 1, top-k 20, top-p 0.95) and alternate greedy and sampled lanes, at + confidence 0 and 0.5, over {1,1}, {16,16,16,16} and {1,1,16,16,16,16} rows with four-draft chains. In every shape + the shared pass's logits equal each lone pass's byte for byte, and the chains' lengths, proposals and probabilities + are equal; the planted broken-gather control is caught. +- `zig build test-cuda-host`: all host tests pass, including the new `drawPick` test. `zig fmt --check` is clean. + `tools/zig/lean_check.py`: 428 problems, the same 428 as 1.1.8 (inherited; none added). Every file stays within + 600 lines. +- Served: in every cell of all four runs below, `--alone` found every concurrent reply's token hash equal to its lone + run's, and `--serial` found every lone run equal to `"draft": false` (0 unequal, 0 failed, 0 unmeasured over the + 64 cells). + +## Speed + +`tools/bench_concurrent.py http://127.0.0.1:30001 Qwen3.8-Flash-Next --levels 1,2,4,5 --tokens 256 --reps 3 +--temperatures 1.0,0 --seed 1234 --alone --serial`, one binary, four runs alternated on one machine in one +session: A = `TF_FLASHNEXT_SAMPLED_HEADS=0` (the previous behaviour), B = the default. Aggregate tok/s, the +median of three repeats; the tool's full reports are [A1.json](A1.json), [A2.json](A2.json), [B1.json](B1.json) and +[B2.json](B2.json). The last column is the two B runs over the two A runs. + +| cell | streams | A1 | A2 | B1 | B2 | B/A | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| code, T=1 | 1 | 66.5 | 67.7 | 67.8 | 67.0 | +0% | +| code, T=1 | 2 | 77.5 | 77.9 | 82.7 | 82.3 | +6% | +| code, T=1 | 4 | 99.9 | 99.9 | 119.2 | 118.4 | +19% | +| code, T=1 | 5 | 107.2 | 107.5 | 132.8 | 132.7 | +24% | +| chat, T=1 | 1 | 41.8 | 42.4 | 42.5 | 42.4 | +1% | +| chat, T=1 | 2 | 60.9 | 61.3 | 65.4 | 65.3 | +7% | +| chat, T=1 | 4 | 82.1 | 82.2 | 94.6 | 94.6 | +15% | +| chat, T=1 | 5 | 87.0 | 87.0 | 103.4 | 103.3 | +19% | +| code, T=0 | 1 | 66.8 | 67.6 | 67.8 | 67.2 | +0% | +| code, T=0 | 2 | 123.5 | 124.3 | 124.5 | 124.4 | +0% | +| code, T=0 | 4 | 210.6 | 211.6 | 212.5 | 210.4 | +0% | +| code, T=0 | 5 | 241.8 | 243.0 | 243.3 | 241.7 | +0% | +| chat, T=0 | 1 | 42.5 | 42.7 | 42.8 | 42.6 | +0% | +| chat, T=0 | 2 | 80.4 | 81.3 | 81.3 | 81.0 | +0% | +| chat, T=0 | 4 | 137.9 | 140.8 | 141.8 | 141.0 | +1% | +| chat, T=0 | 5 | 160.8 | 164.5 | 164.7 | 164.1 | +1% | + +Sampled requests at 4 and 5 streams decode 15% to 24% faster in aggregate, 6% to 7% at 2 streams, and within noise +at 1 stream; greedy cells and time to first token are unchanged within noise (at 5 streams, 249 to 289 ms in every +run). The two runs of each side agree within 1%. + +What remains of the gap between sampled and greedy cells is the host-side sampling work that is still done a lane +at a time: the score-row download and top-k scan for each draft (79,591 columns) and for each verify row +(248,320 columns). Moving those candidates to the device and waiting once a window is separate work. + +## Memory + +Available host memory did not fall below 22.6 GiB during the window. The server's admission and reserve are +unchanged by this change: a sampled lane reuses its own host row buffer, and no device allocation is added. diff --git a/docs/receipts/flashnext-cuda-sampled-heads/commit.txt b/docs/receipts/flashnext-cuda-sampled-heads/commit.txt new file mode 100644 index 0000000000..c836c22980 --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/commit.txt @@ -0,0 +1 @@ +b27b3ec1fae427f0964c95554537dc33e68a2b60 diff --git a/docs/receipts/flashnext-cuda-sampled-heads/parity.txt b/docs/receipts/flashnext-cuda-sampled-heads/parity.txt new file mode 100644 index 0000000000..9c4e89066a --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/parity.txt @@ -0,0 +1,19 @@ +tf-cuda-test flashnext-mtp-parity CHECKPOINT zig/tests/cuda/flashnext/lr7/mid.ids (exit code 0) +PASS a shared MTP head over 2 greedy streams ({ 1, 1 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 4 greedy streams ({ 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 6 greedy streams ({ 1, 1, 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 2 greedy streams ({ 1, 1 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 4 greedy streams ({ 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 6 greedy streams ({ 1, 1, 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 2 sampled streams ({ 1, 1 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 2 mixed streams ({ 1, 1 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 4 sampled streams ({ 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 4 mixed streams ({ 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 6 sampled streams ({ 1, 1, 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 6 mixed streams ({ 1, 1, 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 2 sampled streams ({ 1, 1 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 2 mixed streams ({ 1, 1 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 4 sampled streams ({ 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 4 mixed streams ({ 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 6 sampled streams ({ 1, 1, 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop +PASS a shared MTP head over 6 mixed streams ({ 1, 1, 16, 16, 16, 16 } rows) equals each stream's own pass: logits, proposals, probabilities and the stop From f435921c7df99d9b42597abba182f4031498430c Mon Sep 17 00:00:00 2001 From: JRB AI <197533392+jrb-ai@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:55:26 +1100 Subject: [PATCH 3/5] docs: the receipt names its sides baseline and this PR --- .../flashnext-cuda-sampled-heads/{A1.json => baseline-1.json} | 0 .../flashnext-cuda-sampled-heads/{A2.json => baseline-2.json} | 0 .../flashnext-cuda-sampled-heads/{B1.json => this-pr-1.json} | 0 .../flashnext-cuda-sampled-heads/{B2.json => this-pr-2.json} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename docs/receipts/flashnext-cuda-sampled-heads/{A1.json => baseline-1.json} (100%) rename docs/receipts/flashnext-cuda-sampled-heads/{A2.json => baseline-2.json} (100%) rename docs/receipts/flashnext-cuda-sampled-heads/{B1.json => this-pr-1.json} (100%) rename docs/receipts/flashnext-cuda-sampled-heads/{B2.json => this-pr-2.json} (100%) diff --git a/docs/receipts/flashnext-cuda-sampled-heads/A1.json b/docs/receipts/flashnext-cuda-sampled-heads/baseline-1.json similarity index 100% rename from docs/receipts/flashnext-cuda-sampled-heads/A1.json rename to docs/receipts/flashnext-cuda-sampled-heads/baseline-1.json diff --git a/docs/receipts/flashnext-cuda-sampled-heads/A2.json b/docs/receipts/flashnext-cuda-sampled-heads/baseline-2.json similarity index 100% rename from docs/receipts/flashnext-cuda-sampled-heads/A2.json rename to docs/receipts/flashnext-cuda-sampled-heads/baseline-2.json diff --git a/docs/receipts/flashnext-cuda-sampled-heads/B1.json b/docs/receipts/flashnext-cuda-sampled-heads/this-pr-1.json similarity index 100% rename from docs/receipts/flashnext-cuda-sampled-heads/B1.json rename to docs/receipts/flashnext-cuda-sampled-heads/this-pr-1.json diff --git a/docs/receipts/flashnext-cuda-sampled-heads/B2.json b/docs/receipts/flashnext-cuda-sampled-heads/this-pr-2.json similarity index 100% rename from docs/receipts/flashnext-cuda-sampled-heads/B2.json rename to docs/receipts/flashnext-cuda-sampled-heads/this-pr-2.json From 12606ff7869da924c0de864f22f2724618502a06 Mon Sep 17 00:00:00 2001 From: JRB AI <197533392+jrb-ai@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:56:03 +1100 Subject: [PATCH 4/5] docs: receipt README names its sides baseline and this PR --- docs/receipts/flashnext-cuda-sampled-heads/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/receipts/flashnext-cuda-sampled-heads/README.md b/docs/receipts/flashnext-cuda-sampled-heads/README.md index 3899da6181..37a4cf11e5 100644 --- a/docs/receipts/flashnext-cuda-sampled-heads/README.md +++ b/docs/receipts/flashnext-cuda-sampled-heads/README.md @@ -30,7 +30,7 @@ Server: `tensorfold-native serve CHECKPOINT --context 131072 --parallel 5 --kv-c - `zig build test-cuda-host`: all host tests pass, including the new `drawPick` test. `zig fmt --check` is clean. `tools/zig/lean_check.py`: 428 problems, the same 428 as 1.1.8 (inherited; none added). Every file stays within 600 lines. -- Served: in every cell of all four runs below, `--alone` found every concurrent reply's token hash equal to its lone +- Served: in every cell of the four runs below (baseline and this PR, two each), `--alone` found every concurrent reply's token hash equal to its lone run's, and `--serial` found every lone run equal to `"draft": false` (0 unequal, 0 failed, 0 unmeasured over the 64 cells). @@ -38,11 +38,13 @@ Server: `tensorfold-native serve CHECKPOINT --context 131072 --parallel 5 --kv-c `tools/bench_concurrent.py http://127.0.0.1:30001 Qwen3.8-Flash-Next --levels 1,2,4,5 --tokens 256 --reps 3 --temperatures 1.0,0 --seed 1234 --alone --serial`, one binary, four runs alternated on one machine in one -session: A = `TF_FLASHNEXT_SAMPLED_HEADS=0` (the previous behaviour), B = the default. Aggregate tok/s, the -median of three repeats; the tool's full reports are [A1.json](A1.json), [A2.json](A2.json), [B1.json](B1.json) and -[B2.json](B2.json). The last column is the two B runs over the two A runs. +session: **baseline** is `TF_FLASHNEXT_SAMPLED_HEADS=0`, the behaviour before this change; **this PR** is the change (on +by default); the order was baseline, this PR, baseline, this PR. Aggregate tok/s, the median of three repeats; the +tool's full reports are [baseline-1.json](baseline-1.json), [baseline-2.json](baseline-2.json), +[this-pr-1.json](this-pr-1.json) and [this-pr-2.json](this-pr-2.json). The last column is the two this-PR runs over +the two baseline runs. -| cell | streams | A1 | A2 | B1 | B2 | B/A | +| cell | streams | baseline run 1 | baseline run 2 | this PR run 1 | this PR run 2 | this PR / baseline | | --- | ---: | ---: | ---: | ---: | ---: | ---: | | code, T=1 | 1 | 66.5 | 67.7 | 67.8 | 67.0 | +0% | | code, T=1 | 2 | 77.5 | 77.9 | 82.7 | 82.3 | +6% | From eec960ae0e6e575aa31806d9cccb3aa37b5ce023 Mon Sep 17 00:00:00 2001 From: JRB AI <197533392+jrb-ai@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:57:51 +1100 Subject: [PATCH 5/5] docs: prompt-speed receipt, cold 12k-token prompt on both sides --- .../flashnext-cuda-sampled-heads/README.md | 15 + .../prefill-baseline.json | 282 ++++++++++++++++++ .../prefill-this-pr.json | 282 ++++++++++++++++++ 3 files changed, 579 insertions(+) create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/prefill-baseline.json create mode 100644 docs/receipts/flashnext-cuda-sampled-heads/prefill-this-pr.json diff --git a/docs/receipts/flashnext-cuda-sampled-heads/README.md b/docs/receipts/flashnext-cuda-sampled-heads/README.md index 37a4cf11e5..a0d2c69fe2 100644 --- a/docs/receipts/flashnext-cuda-sampled-heads/README.md +++ b/docs/receipts/flashnext-cuda-sampled-heads/README.md @@ -71,6 +71,21 @@ What remains of the gap between sampled and greedy cells is the host-side sampli at a time: the score-row download and top-k scan for each draft (79,591 columns) and for each verify row (248,320 columns). Moving those candidates to the device and waiting once a window is separate work. +## Prompt speed + +`tools/bench_cache.py http://127.0.0.1:30001 Qwen3.8-Flash-Next --tokens 64 --repeats 2 --streams 2 --require-cold --prompt-file long12k.json`, the prompt being the 12,001 token ids of `zig/tests/cuda/flashnext/lr7/long12k.ids`, a fresh +server for each side so the first request is cold; both runs qualified ([prefill-baseline.json](prefill-baseline.json), +[prefill-this-pr.json](prefill-this-pr.json)). + +| request (12,001-token prompt, 64 tokens) | baseline | this PR | +| --- | ---: | ---: | +| cold first request, `ttft_s` (mean of the two cold requests) | 7.22 s (7.39, 7.05) | 7.22 s (7.29, 7.14) | +| prompt tokens over that time | 1,662 tok/s | 1,663 tok/s | +| repeated request on the kept prompt, `ttft_s` | 0.088 s | 0.092 s | +| two concurrent requests on the kept prompt, `ttft_s` | 0.084 s, 0.258 s | 0.091 s, 0.238 s | +| decode, first request | 39.8 tok/s | 39.5 tok/s | +| repeated and concurrent replies equal to the first | 4 of 4 | 4 of 4 | + ## Memory Available host memory did not fall below 22.6 GiB during the window. The server's admission and reserve are diff --git a/docs/receipts/flashnext-cuda-sampled-heads/prefill-baseline.json b/docs/receipts/flashnext-cuda-sampled-heads/prefill-baseline.json new file mode 100644 index 0000000000..306811a717 --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/prefill-baseline.json @@ -0,0 +1,282 @@ +{ + "schema_version": 1, + "fixture_sha256": "408413104f58d6254b9b8dfb1749374676fc2b4ef95c282e181c54f040a7844c", + "requested_reply_tokens": 64, + "temperature": 0, + "seed": 1234, + "rows": [ + { + "phase": "first", + "prompt_tokens": 12001, + "cached_tokens": 0, + "tokens": 64, + "cache_state": "cold", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 7.3930314049976005, + "decode_tps": 39.78515666801017, + "elapsed_s": 8.986276678999275, + "chunk_offsets_s": [ + 7.3930314049976005, + 7.452872735997516, + 7.504984362996765, + 7.578640725998412, + 7.653881192996778, + 7.713859007999417, + 7.770379894998769, + 7.82038073899821, + 7.882158012998843, + 7.960401420998096, + 8.039778369999112, + 8.117834467997454, + 8.185093921998487, + 8.234962539998378, + 8.296530093997717, + 8.354817911997088, + 8.397964866999246, + 8.444689097999799, + 8.498634809999203, + 8.548625377999997, + 8.595349657000042, + 8.642001739997795, + 8.687673138996615, + 8.733316578996892, + 8.779031981997832, + 8.825096650998603, + 8.87106664699968, + 8.917356291000033, + 8.976536542999384 + ], + "chunk_gaps_ms": { + "count": 28, + "p50": 53.0286694993265, + "p95": 78.24340799925267, + "max": 79.37694900101633 + } + }, + { + "phase": "repeated", + "prompt_tokens": 12001, + "cached_tokens": 0, + "tokens": 64, + "cache_state": "cold", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 7.0488765299996885, + "decode_tps": 39.59256214358831, + "elapsed_s": 8.640207095999358, + "chunk_offsets_s": [ + 7.0488765299996885, + 7.103205569997954, + 7.148554937997687, + 7.207939895997697, + 7.279064713999105, + 7.335998608999944, + 7.397234059997572, + 7.453000942998187, + 7.522500251998281, + 7.604729623999447, + 7.681843256999855, + 7.757238674999826, + 7.82124263500009, + 7.868935934999172, + 7.9370556689973455, + 8.001258685999346, + 8.047658436000347, + 8.092747672999394, + 8.137879114998213, + 8.184471882999787, + 8.231473010997433, + 8.276970797000104, + 8.32259456599786, + 8.37128683499759, + 8.426174371998059, + 8.479653538997809, + 8.531781549998414, + 8.58130751799763, + 8.640084488997672 + ], + "chunk_gaps_ms": { + "count": 28, + "p50": 54.60828849936661, + "p95": 77.1136330004083, + "max": 82.22937200116576 + }, + "equal_first_tokens": true, + "equal_first_output": true + }, + { + "phase": "repeated", + "prompt_tokens": 12001, + "cached_tokens": 12000, + "tokens": 64, + "cache_state": "reused", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 0.08847288999822922, + "decode_tps": 39.831433222863154, + "elapsed_s": 1.6702309459979006, + "chunk_offsets_s": [ + 0.08847288999822922, + 0.14446260499971686, + 0.18973142499817186, + 0.25048512399735046, + 0.3320610989976558, + 0.3951890589996765, + 0.4491361529981077, + 0.49635851600032765, + 0.5571502940001665, + 0.6366665440000361, + 0.7256390099973942, + 0.8035744980006712, + 0.8745152119990962, + 0.9254885410009592, + 0.984912585998245, + 1.0417139889977989, + 1.0857917989997077, + 1.131667142999504, + 1.1771184379977058, + 1.2227321899990784, + 1.2702219689999765, + 1.3234463989974756, + 1.3784695979993558, + 1.4271067189984024, + 1.4725401699979557, + 1.5194703019988083, + 1.5663388670000131, + 1.6122675359983987, + 1.6701382959981856 + ], + "chunk_gaps_ms": { + "count": 28, + "p50": 53.585761997965164, + "p95": 81.57597500030533, + "max": 88.97246599735809 + }, + "equal_first_tokens": true, + "equal_first_output": true + }, + { + "phase": "concurrent", + "prompt_tokens": 12001, + "cached_tokens": 12000, + "tokens": 64, + "cache_state": "reused", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 0.25757561099817394, + "decode_tps": 33.529243800068514, + "elapsed_s": 2.1366379510000115, + "chunk_offsets_s": [ + 0.25757561099817394, + 0.3300344419985777, + 0.40214698300042073, + 0.47073326499958057, + 0.5451408439985244, + 0.6196030399987649, + 0.6913105919993541, + 0.7567946439994557, + 0.8193167659992469, + 0.8959531079999579, + 0.9853946119983448, + 1.0800884929994936, + 1.1814040179997392, + 1.2611190819989133, + 1.3208363759986241, + 1.3849888469994767, + 1.4415295830003743, + 1.4928696209972259, + 1.5438894769977196, + 1.5959687309987203, + 1.6469767710004817, + 1.6988396419983474, + 1.750532262998604, + 1.8011590149981203, + 1.8526276079974195, + 1.9057587440001953, + 1.956387652997364, + 2.0072609529997862, + 2.074207129997376, + 2.1365323919999355 + ], + "chunk_gaps_ms": { + "count": 29, + "p50": 62.522121999791125, + "p95": 94.69388100114884, + "max": 101.31552500024554 + }, + "equal_first_tokens": true, + "equal_first_output": true + }, + { + "phase": "concurrent", + "prompt_tokens": 12001, + "cached_tokens": 12000, + "tokens": 64, + "cache_state": "reused", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 0.0837184129995876, + "decode_tps": 31.662880347363807, + "elapsed_s": 2.0735446809994755, + "chunk_offsets_s": [ + 0.0837184129995876, + 0.14112451600158238, + 0.3299749560028431, + 0.4020859129996097, + 0.47104783200120437, + 0.5449693510017823, + 0.6199161340009596, + 0.6915090809998219, + 0.757147893000365, + 0.8195128150000528, + 0.8962065790001361, + 0.9856568899995182, + 1.080315352002799, + 1.181604931000038, + 1.2613517710015003, + 1.3210545069996442, + 1.3852742439994472, + 1.4419258199995966, + 1.492907407002349, + 1.5442295880020538, + 1.596303196001827, + 1.6470050570023886, + 1.6987499669994577, + 1.7503516670003592, + 1.801493623002898, + 1.8529137340010493, + 1.9060830440030259, + 1.956737961001636, + 2.0074597420025384, + 2.073430003001704 + ], + "chunk_gaps_ms": { + "count": 29, + "p50": 62.364921999687795, + "p95": 101.289578997239, + "max": 188.85044000126072 + }, + "equal_first_tokens": true, + "equal_first_output": true + } + ], + "concurrent_timing": { + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 2.136758094002289, + "end_to_end_tps": 59.903832988528684, + "timing_valid": true + }, + "qualified": true +} diff --git a/docs/receipts/flashnext-cuda-sampled-heads/prefill-this-pr.json b/docs/receipts/flashnext-cuda-sampled-heads/prefill-this-pr.json new file mode 100644 index 0000000000..b7cbf131aa --- /dev/null +++ b/docs/receipts/flashnext-cuda-sampled-heads/prefill-this-pr.json @@ -0,0 +1,282 @@ +{ + "schema_version": 1, + "fixture_sha256": "408413104f58d6254b9b8dfb1749374676fc2b4ef95c282e181c54f040a7844c", + "requested_reply_tokens": 64, + "temperature": 0, + "seed": 1234, + "rows": [ + { + "phase": "first", + "prompt_tokens": 12001, + "cached_tokens": 0, + "tokens": 64, + "cache_state": "cold", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 7.2880335850022675, + "decode_tps": 39.49289359355985, + "elapsed_s": 8.894180505001714, + "chunk_offsets_s": [ + 7.2880335850022675, + 7.347317089999706, + 7.39444468000147, + 7.45971475099941, + 7.5357721140026115, + 7.598715291002009, + 7.6561383540029055, + 7.706699774000299, + 7.771416706000309, + 7.851185621002514, + 7.933808023000893, + 8.019871374999639, + 8.090753878001124, + 8.142275345002417, + 8.205475169001147, + 8.264787661002629, + 8.308588289000909, + 8.35561093800061, + 8.403322645001026, + 8.450011078999523, + 8.49642846000279, + 8.543793406002806, + 8.590362028000527, + 8.636956718000874, + 8.683540191002976, + 8.729991434000112, + 8.775978311001381, + 8.822740981002426, + 8.883257289002358 + ], + "chunk_gaps_ms": { + "count": 28, + "p50": 51.041443499343586, + "p95": 82.62240199837834, + "max": 86.06335199874593 + } + }, + { + "phase": "repeated", + "prompt_tokens": 12001, + "cached_tokens": 0, + "tokens": 64, + "cache_state": "cold", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 7.143387772997812, + "decode_tps": 40.24330216409396, + "elapsed_s": 8.708939880998514, + "chunk_offsets_s": [ + 7.143387772997812, + 7.204377252000995, + 7.2518365169999015, + 7.313483273999736, + 7.3858276789978845, + 7.4455154729985225, + 7.501530333000119, + 7.549979890998657, + 7.611399197998253, + 7.691093202000047, + 7.7709054599981755, + 7.847780857999169, + 7.9128999060012575, + 7.960520534001262, + 8.022493404998386, + 8.079517304999172, + 8.128784767999605, + 8.179405022998253, + 8.226605888998165, + 8.274524954998924, + 8.322680786001001, + 8.369154624000657, + 8.415615947000333, + 8.461313260999304, + 8.509309721997852, + 8.554961679998087, + 8.60147403999872, + 8.64763883699925, + 8.708865668999351 + ], + "chunk_gaps_ms": { + "count": 28, + "p50": 49.94385899954068, + "p95": 79.69400400179438, + "max": 79.81225799812819 + }, + "equal_first_tokens": true, + "equal_first_output": true + }, + { + "phase": "repeated", + "prompt_tokens": 12001, + "cached_tokens": 12000, + "tokens": 64, + "cache_state": "reused", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 0.09238847300002817, + "decode_tps": 40.17553051497822, + "elapsed_s": 1.6605816090013832, + "chunk_offsets_s": [ + 0.09238847300002817, + 0.14944707400354673, + 0.19598270500137005, + 0.2571114790007414, + 0.3298653710007784, + 0.38950783600012073, + 0.45139565800127457, + 0.502751130003162, + 0.5658726310030033, + 0.6452148640019004, + 0.7245377480030584, + 0.8005558990007557, + 0.8653492400007963, + 0.9145561260011164, + 0.9772003900034179, + 1.0349949740011652, + 1.0792223700009345, + 1.1261517610000737, + 1.1725234980003734, + 1.219262949001859, + 1.2661181120020046, + 1.3130137020016264, + 1.3594776870013447, + 1.410310950002895, + 1.4592962720016658, + 1.5071230150024348, + 1.5535752200012212, + 1.6003697930027556, + 1.6605071560006763 + ], + "chunk_gaps_ms": { + "count": 28, + "p50": 51.09436750171881, + "p95": 79.32288400115795, + "max": 79.34223299889709 + }, + "equal_first_tokens": true, + "equal_first_output": true + }, + { + "phase": "concurrent", + "prompt_tokens": 12001, + "cached_tokens": 12000, + "tokens": 64, + "cache_state": "reused", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 0.09146410699759144, + "decode_tps": 31.831605769011226, + "elapsed_s": 2.070701126998756, + "chunk_offsets_s": [ + 0.09146410699759144, + 0.14843915299934451, + 0.30506025700015016, + 0.367017341999599, + 0.4359905189994606, + 0.5111868779968063, + 0.5871719529968686, + 0.6596784939974896, + 0.7272491209987493, + 0.7990144489995146, + 0.8797677699985798, + 0.9715852519984765, + 1.0673460959988006, + 1.166211742998712, + 1.2420027809966996, + 1.303376230996946, + 1.3664326729995082, + 1.4230617499997607, + 1.473918034000235, + 1.5256906909999088, + 1.5775839099987934, + 1.6297333299989987, + 1.6823805869971693, + 1.7387147329973232, + 1.7935931519969017, + 1.84776385999794, + 1.9000263629968686, + 1.9519631089970062, + 2.003851854999084, + 2.0706291059977957 + ], + "chunk_gaps_ms": { + "count": 29, + "p50": 61.95708499944885, + "p95": 98.8656469999114, + "max": 156.62110400080564 + }, + "equal_first_tokens": true, + "equal_first_output": true + }, + { + "phase": "concurrent", + "prompt_tokens": 12001, + "cached_tokens": 12000, + "tokens": 64, + "cache_state": "reused", + "complete": true, + "token_sha": "b02743b3077d", + "token_sha256": "b02743b3077db92948d38095e62518a407b1f01de2ba199833a2aea763db26e6", + "output_sha256": "6ad4c3835a3054e1655c59643034015ce7c50d24b96b988b4fcb21a0efcb18f4", + "ttft_s": 0.2380453200021293, + "decode_tps": 33.243040889818516, + "elapsed_s": 2.1333641090022866, + "chunk_offsets_s": [ + 0.2380453200021293, + 0.30549311299910187, + 0.3669099370017648, + 0.4363510990006034, + 0.5119046460022219, + 0.5876175010016595, + 0.6603486000021803, + 0.7275718670025526, + 0.7995085860020481, + 0.8801383860009082, + 0.9725490780001564, + 1.0676685069993255, + 1.1666734739992535, + 1.2424281170024187, + 1.3035808650020044, + 1.366306332001841, + 1.4234008060011547, + 1.4741764110003714, + 1.52608132900059, + 1.5781632580001315, + 1.6302694020014314, + 1.682767304002482, + 1.7386017400021956, + 1.7938901160014211, + 1.8477282320018276, + 1.900251308001316, + 1.952367353002046, + 2.004308995001338, + 2.07184128100198, + 2.1331788070019684 + ], + "chunk_gaps_ms": { + "count": 29, + "p50": 61.416824002662906, + "p95": 95.11942899916903, + "max": 99.004966999928 + }, + "equal_first_tokens": true, + "equal_first_output": true + } + ], + "concurrent_timing": { + "requested_requests": 2, + "timed_requests": 2, + "makespan_s": 2.1333641090022866, + "end_to_end_tps": 59.99913444679724, + "timing_valid": true + }, + "qualified": true +}